Ë
    M²jWI  ã            	       óÜ  — U d dl Z d dlZd dlmZmZ d dlmZ d dlmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d
dlmZmZmZmZmZmZmZ d
dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d
dl'm(Z(m)Z) dgZ*e+e,   e-d<   de#de#de.fd„Z/de+e#   de.fd„Z0de+e#   de+e#   de+e#   fd„Z1de
jd                  defd„Z3dedefd„Z4dedede$fd„Z5de,dede%fd„Z6de,dedede%fd „Z7de,de
jd                  de%fd!„Z8de,d"efd#„Z9d$„ Z:d%„ Z;de,d&ed'e+e   de+e"   fd(„Z<d)ede#fd*„Z=de,d+ede+e%   fd,„Z>dedefd-„Z?de
jd                  de+e   fd.„Z@de,d/ed0ede+e"   fd1„ZAd)eBe,ef   defd2„ZCd3ed4ed5ed6efd7„ZDy)8é    N)Úbisect_rightÚinsort)ÚCallable)ÚAnyÚcast)Ú_get_device_module)ÚShardMetadata)ÚShardedTensor)ÚDTensor)Ú%compute_local_shape_and_global_offseté   )ÚBytesStorageMetadataÚChunkStorageMetadataÚMetadataIndexÚSTATE_DICT_TYPEÚSTORAGE_TYPESÚTensorPropertiesÚTensorStorageMetadata)ÚLoadItemTypeÚReadItemÚSavePlanÚTensorWriteDataÚ	WriteItemÚWriteItemType)Ú"_check_shard_metadata_pair_overlapÚ+_shards_get_overlap_region_wrt_saved_tensorÚ create_read_items_for_chunk_listÚ__all__ÚplanÚ
other_planÚreturnc                 ó"  — | j                   |j                   k7  ryt        | j                  «      t        |j                  «      k7  ryt        | j                  |j                  «      D �]%  \  }}|j                  |j                  k7  r y|j
                  }|j
                  }|j                  |j                  k7  s2|j                  |j                  k7  s|j
                  |j
                  k7  r y|j                  }|j                  }|r|r|s|r y|sŒ¬|sŒ¯|j                  |j                  k7  r y|j                  }|j                  }	|r|	r|s|	r y|sŒï|	sŒò|j                  |	j                  k7  s|j                  |	j                  k7  s�Œ& y y)a  
    Compare the two Save plans and return True if they are equal.

    Args:
        plan (SavePlan): First SavePlan to compare.
        other_plan (SavePlan): Second SavePlan to compare.

    Returns:
       True if the two plans are equal, False otherwise.
    FT)ÚusableÚlenÚitemsÚzipÚtypeÚindexÚfqnÚoffsetÚtensor_dataÚsizeÚchunkÚoffsetsÚsizes)
r   r    Ú	plan_itemÚother_plan_itemÚplan_metadata_indexÚother_plan_metadata_indexr+   Úother_tensor_datar-   Úother_chunks
             úk/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/planner_helpers.pyÚ_compare_save_plansr7   *   sn  € ð ‡{�{�j×'Ñ'Ò'Øô ˆ4�:‰:ƒœ#˜j×.Ñ.Ó/Ò/Øô '*¨$¯*©*°j×6FÑ6FÓ&Gó )!Ñ"ˆ	�?à�>‰>˜_×1Ñ1Ò1Ùà'Ÿo™oÐØ$3×$9Ñ$9Ð!ð  ×#Ñ#Ð'@×'DÑ'DÒDØ"×)Ñ)Ð-F×-MÑ-MÒMØ"×(Ñ(Ð,E×,KÑ,KÒKáð  ×+Ñ+ˆØ+×7Ñ7ÐÙÑ 1ÙÑ 1áâÒ,à×ÑÐ#4×#9Ñ#9Ò9Ùð  ×%Ñ%ˆEØ+×1Ñ1ˆKÙ™k±5¹[Ùò šà—M‘M [×%8Ñ%8Ò8Ø—{‘{ k×&7Ñ&7Ô7á ðS)!ðV ó    Údelta_plansc                 ó&   — t        d„ | D «       «      S )zæ
    Check if any delta plan is usable, indicating the plan has changed.

    Args:
        delta_plans (List[SavePlan]): A list of delta plans to check.
    Returns:
        True if any delta plan is usable, False otherwise.
    c              3   ó<   K  — | ]  }|xr |j                   –— Œ y ­w©N)r#   )Ú.0Ú
delta_plans     r6   ú	<genexpr>z(_contains_usable_plan.<locals>.<genexpr>t   s   è ø€ ÒN°JˆzÒ/˜j×/Ñ/Ó/ÑNùs   ‚)Úany)r9   s    r6   Ú_contains_usable_planrA   k   s   € ô ÑNÀ+ÔNÓNÐNr8   Úcached_plansc                 ó”   — g }t        | |«      D ]6  \  }}|r|j                  s|j                  |«       Œ&|j                  |«       Œ8 |S )ac  
    Merge a list of delta plans into a single plan.

    Args:
        cached_plans (List[SavePlan]): A list of cached plans.
        delta_plans (List[SavePlan]): A list of delta plans to merge. It can contain empty plans

    Returns:
        A single merged plan. If a delta plan is not usable, use the cached plan. Otherwise, use the delta plan.
    )r&   r#   Úappend)rB   r9   Úmerged_plansÚcached_planr>   s        r6   Ú_merge_delta_local_plansrG   w   sS   € ð €Lä#& |°[Ó#Aò ,Ñˆ�ZÙ˜j×/Ò/Ø×Ñ Õ,à×Ñ 
Õ+ð	,ð Ðr8   Útensorc           	      ó”   — t        t        j                  dgt        | j	                  «       «      z  «      | j	                  «       ¬«      S )Nr   ©r.   r/   )r   ÚtorchÚSizer$   r,   )rH   s    r6   Ú_create_chunk_from_tensorrM   �   s5   € ÜÜ—
‘
˜A˜3¤ V§[¡[£]Ó!3Ñ3Ó4¸F¿K¹K»Môð r8   Úshard_mdc                 ó�   — t        t        j                  | j                  «      t        j                  | j                  «      ¬«      S ©NrJ   )r   rK   rL   Úshard_offsetsÚshard_sizes)rN   s    r6   Ú_chunk_for_shardrS   –   s3   € ÜÜ—
‘
˜8×1Ñ1Ó2Ü�j‰j˜×-Ñ-Ó.ôð r8   Úsharded_tensorc                 ó  — | j                  «       j                  }t        |j                  |j                  |j
                  |j                  |j                  ¬«      }t        t        |«      || j                  «       j                  ¬«      S )N)ÚdtypeÚlayoutÚrequires_gradÚmemory_formatÚ
pin_memory©r-   Ú
propertiesr,   )ÚmetadataÚtensor_propertiesr   rV   rW   rX   rY   rZ   r   rS   r,   )rT   rN   Úshard_propertiesr\   s       r6   Ú_sharded_tensor_metadatar`   �   s|   € ð &×.Ñ.Ó0×BÑBÐä!Ø×$Ñ$Ø×&Ñ&Ø&×4Ñ4Ø&×4Ñ4Ø#×.Ñ.ô€Jô Ü˜xÓ(ØØ×$Ñ$Ó&×+Ñ+ôð r8   r)   c                 óŠ  — t        |j                  |j                  |j                  «      \  }}t	        j
                  |«      t	        j
                  |«      }}t        t        | |«      t        j                  t        t        ||¬«      t        j                  |j                  «       «      |j                  «       ¬«      ¬«      S )NrJ   r[   ©r(   r'   r+   )r   ÚshapeÚdevice_meshÚ
placementsrK   rL   r   r   r   ÚSHARDr   r   r   Úcreate_from_tensorÚto_localr,   )r)   rH   r/   r.   s       r6   Ú_create_write_items_for_dtensorri   ±   sš   € Ü:Ø�‰�f×(Ñ(¨&×*;Ñ*;ó�N€Eˆ7ô —Z‘Z Ó&¬¯
©
°7Ó(;ˆ7€EäÜ˜C Ó)Ü× Ñ Ü#Ü&ØØôô (×:Ñ:¸6¿?¹?Ó;LÓMØ—‘“ô
ôð r8   c                 ó    — t        j                  |j                  «      }t        t	        | |«      t
        j                  t        ||«      ¬«      S )Nrb   )rK   rL   rQ   r   r   r   rf   r`   )r)   rT   rN   r.   s       r6   Ú_create_write_item_for_shardrk   Å   sB   € ô �j‰j˜×/Ñ/Ó0€GÜÜ˜C Ó)Ü× Ñ Ü,¨^¸XÓFôð r8   c                 ó:  — t        j                  dgt        |j                  «       «      z  «      }t	        t        | |«      t        j                  t        t        ||j                  «       ¬«      t        j                  |«      |j                  «       ¬«      ¬«      S )Nr   rJ   r[   rb   )rK   rL   r$   r,   r   r   r   ÚTENSORr   r   r   rg   )r)   rH   r.   s      r6   Ú_create_write_item_for_tensorrn   Ð   sq   € Ü�j‰j˜!˜œs 6§;¡;£=Ó1Ñ1Ó2€GÜÜ˜C Ó)Ü×!Ñ!Ü#Ü&¨w¸f¿k¹k»mÔLÜ'×:Ñ:¸6ÓBØ—‘“ô
ôð r8   Úbytesc                 óJ   — t        t        | «      t        j                  ¬«      S )N)r(   r'   )r   r   r   ÚBYTE_IO)r)   ro   s     r6   Ú_create_write_item_for_bytesiorr   Ý   s    € ÜÜ˜CÓ Ü×"Ñ"ôð r8   c           
      ó¸   — t        t        j                  | t        j                  |f«      |t        j                  |f«      t        j                  |f«      ¬«      S ©N)r'   Ú
dest_indexÚdest_offsetsÚstorage_indexÚstorage_offsetsÚlengths)r   r   rq   rK   rL   ©ru   Údest_offsetrw   Ústorage_offsetÚlengths        r6   Ú_create_read_item_for_byteior~   ä   sK   € ô Ü×!Ñ!ØÜ—Z‘Z  Ó/Ø#ÜŸ
™
 NÐ#4Ó5Ü—
‘
˜F˜9Ó%ôð r8   c           
      ó²   — t        t        j                  | t        j                  |«      |t        j                  |«      t        j                  |«      ¬«      S rt   )r   r   rm   rK   rL   ©ru   rv   rw   rx   ry   s        r6   Ú_create_read_item_for_tensorr�   ñ   sD   € ô Ü× Ñ ØÜ—Z‘Z Ó-Ø#ÜŸ
™
 ?Ó3Ü—
‘
˜7Ó#ôð r8   Úcheckpoint_mdÚlocal_chunksc                 ó  ‡ ‡!‡"— g }|j                   }|r|s|S t        |d   j                  «      }d}|dkD  rDd}t        |«      D ]4  Š t	        ˆ fd„t        j                  ||«      D «       «      }||kD  sŒ1|}‰ }Œ6 |dk(  rdgt        |«      z  Š"dgt        |«      z  Š!nx|D �	cg c]0  }	|	j                  |   |	j                  |   |	j                  |   z   f‘Œ2 c}	Š"|D �	cg c]0  }	|	j                  |   |	j                  |   |	j                  |   z   f‘Œ2 c}	Š!t        t        t        |«      «      ˆ"fd„¬«      }
t        t        t        |«      «      ˆ!fd„¬«      }g }d}t        |
«      }|D �]  }||   }‰!|   \  }}t        ||df«      }|r|d	|…= ||k  r1|
|   }||   }‰"|   \  }}||k\  rnt        |||f«       |dz  }||k  rŒ1|D ]°  \  }}||   }t        ||«      sŒg }g }g }t        ||¬
«      D ]:  \  }}}}|j                  |«       |j                  |«       |j                  |«       Œ< |j                  t        t        | |j                  |«      |t        | |j                  |«      ||¬«      «       Œ² �Œ |S c c}	w c c}	w )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    r   r   c              3   ó\   •K  — | ]#  }|j                   ‰   |j                  ‰   z   –— Œ% y ­wr<   rJ   )r=   r-   Údims     €r6   r?   z3create_read_items_for_chunk_list.<locals>.<genexpr>   s0   øè ø€ ò àð —‘˜cÑ" U§[¡[°Ñ%5Õ5ñùs   ƒ),)r   r   c                 ó   •— ‰|    d   S ©Nr   © )ÚidxÚsaved_boundss    €r6   ú<lambda>z2create_read_items_for_chunk_list.<locals>.<lambda>9  ó   ø€ ˜ SÑ)¨!Ñ,€ r8   )Úkeyc                 ó   •— ‰|    d   S rˆ   r‰   )rŠ   Úlocal_boundss    €r6   rŒ   z2create_read_items_for_chunk_list.<locals>.<lambda>=  r�   r8   éÿÿÿÿN)Úsaved_shardÚcurrent_shardr€   )Úchunksr$   r.   ÚrangeÚmaxÚ	itertoolsÚchainr/   Úsortedr   r   r   r   rD   r�   r   )#r)   r‚   rƒ   Ú
read_itemsÚsaved_chunksÚnum_dimsÚ	sweep_dimÚmax_sizeÚdim_sizeÚcÚsaved_sorted_indicesÚlocal_sorted_indicesÚactive_savedÚ	saved_ptrÚ	num_savedÚ	local_idxÚlocal_chunkÚlocal_startÚ	local_endÚcutoffÚstorage_idxÚstorage_chunkÚsaved_startÚ	saved_endÚ_rx   rv   ry   Ú_dimÚoffset_for_saved_tensorÚoffset_for_current_tensorr}   r†   r�   r‹   s#                                   @@@r6   r   r   þ   s  ú€ ð* "$€JØ ×'Ñ'€Lá™|ØÐä�< ‘?×*Ñ*Ó+€Hð €IØ�!‚|ØˆÜ˜“?ò 	 ˆCÜó ä&Ÿ_™_¨\¸<ÓHôó ˆHð ˜(Ó"Ø#�Ø‘	ð	 ð �1‚}Ø�x¤# lÓ"3Ñ3ˆØ�x¤# lÓ"3Ñ3‰ð "ö
àð �Y‰Y�yÑ! 1§9¡9¨YÑ#7¸!¿'¹'À)Ñ:LÑ#LÒMò
ˆð "ö
àð �Y‰Y�yÑ! 1§9¡9¨YÑ#7¸!¿'¹'À)Ñ:LÑ#LÒMò
ˆô
 "ÜŒc�,ÓÓ Û,ôÐô "ÜŒc�,ÓÓ Û,ôÐð
 +-€LØ€IÜÐ(Ó)€Ià)ó 1ˆ	Ø" 9Ñ-ˆØ!-¨iÑ!8Ñˆ�Yä˜l¨[¸"Ð,=Ó>ˆÙØ˜W˜f˜WÐ%à˜)Ò#Ø.¨yÑ9ˆKØ(¨Ñ5ˆMØ%1°+Ñ%>Ñ"ˆK˜à˜iÒ'Øä�< )¨[Ð!9Ô:Ø˜‰NˆIð ˜)Ó#ð +ò 	‰NˆAˆ{Ø(¨Ñ5ˆMÜ5°kÀ=ÔQØà ˆOØˆLØˆGô =Ø)¸ôò
'ñ ØØ'Ø)Øð  ×&Ñ&Ð'>Ô?Ø×#Ñ#Ð$=Ô>Ø—‘˜vÕ&ð
'ð ×ÑÜ,Ü,¨S°+×2EÑ2EÀyÓQØ!-Ü"/Ø˜]×2Ñ2°Kó#ð %4Ø#ôõ
ò)	ð'1ðd ÐùòQ
ùò
s   Â#5JÃ5JÚ
state_dictc                 óÒ  ‡‡— g }| j                  «       D ]Æ  \  ŠŠt        ‰t        «      r|j                  t	        ‰‰«      «       Œ2t        ‰t
        «      r4|j                  ˆˆfd„‰j                  «       j                  D «       «       Œvt        ‰t        j                  «      r|j                  t        ‰‰«      «       Œ¬|j                  t        ‰‰«      «       ŒÈ t        |«      S )Nc              3   ó8   •K  — | ]  }t        ‰‰|«      –— Œ y ­wr<   )rk   )r=   rN   r)   Úobjs     €€r6   r?   z5_create_default_metadata_only_plan.<locals>.<genexpr>  s"   øè ø€ ò àô -¨S°#°x×@ñùs   ƒ)r%   Ú
isinstancer   rD   ri   r
   Úextendr]   Úshards_metadatarK   ÚTensorrn   rr   r   )r³   Úrequestsr)   r¶   s     @@r6   Ú"_create_default_metadata_only_planr¼   y  s¸   ù€ Ø€HØ×$Ñ$Ó&ò F‰ˆˆSÜ�cœ7Ô#Ø�O‰OÔ;¸CÀÓEÕFÜ˜œ]Ô+Ø�O‰Oô à #§¡£× >Ñ >ôõ ô ˜œUŸ\™\Ô*Ø�O‰OÔ9¸#¸sÓCÕDà�O‰OÔ:¸3ÀÓDÕEðFô �HÓÐr8   Úobjectc                 ó6  — t        |d«      r|j                  | |«      S t        |t        «      r3|j	                  «       D �cg c]  }t        | ||j                  «      ‘Œ c}S t        |t        j                  «      rt        | |«      gS t        | |«      gS c c}w )NÚ__create_write_items__)Úhasattrr¿   r·   r
   Úlocal_shardsrk   r]   rK   rº   rn   rr   )r)   r½   Úshards      r6   Ú_create_write_itemsrÃ   Š  s’   € ÜˆvÐ/Ô0à×,Ñ,¨S°&Ó9Ð9Ü	�FœMÔ	*ð  ×,Ñ,Ó.ö
àô )¨¨f°e·n±nÕEò
ð 	
ô 
�FœEŸL™LÔ	)Ü-¨c°6Ó:Ð;Ð;ä.¨s°FÓ;Ð<Ð<ùò
s   ÁBc                 óÌ   — t        | j                  | j                  | j                  «      \  }}t	        j
                  |«      t	        j
                  |«      }}t        ||¬«      S rP   )r   rc   rd   re   rK   rL   r   )rH   r/   r.   s      r6   Ú_create_chunk_from_dtensorrÅ   ™  sW   € Ü:Ø�‰�f×(Ñ(¨&×*;Ñ*;ó�N€Eˆ7ô —Z‘Z Ó&¬¯
©
°7Ó(;ˆ7€EÜØØôð r8   c                 óJ  — t        | d«      r| j                  «       }|S t        | t        «      r2| j	                  «       D �cg c]  }t        |j                  «      ‘Œ }}|S t        | t        j                  «      rt        | «      g}|S t        dt        | «      › �«      ‚c c}w )NÚ__create_chunk_list__zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )rÀ   rÇ   r·   r
   rÁ   rS   r]   rK   rº   rM   Ú
ValueErrorr'   )rH   rƒ   rÂ   s      r6   Ú_create_chunk_listrÉ   ¤  s°   € ÜˆvÐ.Ô/à×3Ñ3Ó5ˆð Ðô 
�FœMÔ	*à:@×:MÑ:MÓ:Oö
Ø16Ô˜UŸ^™^Õ,ð
ˆð 
ð Ðô 
�FœEŸL™LÔ	)Ü1°&Ó9Ð:ˆð Ðô ðÜ˜V›�~ð'ó
ð 	
ùò
s   ÁB Úmdr¶   c                 óú   — t        |t        «      s	 t        |«      }t        | ||«      S t        t        | «      dt        | «      dd¬«      gS # t        $ r$}t        d| › d�dt	        |«      › �z   «      |‚d }~ww xY w)Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rz   )r·   r   rÉ   rÈ   r'   r   r~   r   )r)   rÊ   r¶   rƒ   Úexs        r6   Ú_create_read_itemsrÍ   ·  sž   € Ü�bÔ.Ô/ð	Ü-¨cÓ2ˆLô 0°°R¸ÓFÐFô )Ü(¨Ó-ØÜ+¨CÓ0Ø Øôð
ð 	
øô ò 	ÜØ2°3°%°rÐ:Ø<¼TÀ"»X¸JÐGñHóð ðûð	ús   ’A Á	A:ÁA5Á5A:c                 óp   — dt         fd„}dt        fd„}dt        j                  fd„}t	        | |||«       y)zP
    Initializes meta tensor if the meta tensor is DTensor or torch.Tensor.
    Úvaluec                 óä  — t        | dd «      }|t        j                  d«      k(  rÊt        j                  j                  «       j                  }t        t        j                  t        |«      j                  «       «      }t        j                  | j                  «       |¬«      }t        j                  || j                  | j                  | j!                  «       | j#                  «       ¬«      }|S | S )NÚdeviceÚmeta©rÑ   )rd   re   rc   Ústride)ÚgetattrrK   rÑ   ÚdistÚdistributed_c10dÚ_get_pg_default_devicer'   r   r   Úcurrent_deviceÚ
empty_likerh   r   Ú
from_localrd   re   r,   rÔ   )rÏ   rÑ   Údevice_typeÚnew_local_tensorÚdtensors        r6   Údtensor_funcz&_init_state_dict.<locals>.dtensor_funcÓ  s¾   € Ü˜ ¨$Ó/ˆØ”U—\‘\ &Ó)Ò)Ü×/Ñ/×FÑFÓH×MÑMˆKÜÜ—‘Ô0°Ó=×LÑLÓNóˆFô  %×/Ñ/°·±Ó0@ÈÔPÐô ×(Ñ(Ø Ø!×-Ñ-Ø ×+Ñ+Ø—j‘j“lØ—|‘|“~ôˆGð ˆNàˆLr8   c                 ó€   — t        | dd «      }|t        j                  d«      k(  rt        dt	        | «      › d�«      ‚| S )NrÑ   rÒ   zFound unsupported type z for meta device loading.)rÕ   rK   rÑ   ÚRuntimeErrorr'   )rÏ   rÑ   s     r6   Úsharded_tensor_funcz-_init_state_dict.<locals>.sharded_tensor_funcè  sF   € Ü˜ ¨$Ó/ˆØ”U—\‘\ &Ó)Ò)ÜØ)¬$¨u«+¨Ð6OÐPóð ð ˆLr8   c                 ó4  — t        | dd «      }|t        j                  d«      k(  rrt        j                  j                  «       j                  }t        t        j                  t        |«      j                  «       «      }t        j                  | |¬«      }|S | S )NrÑ   rÒ   rÓ   )rÕ   rK   rÑ   rÖ   r×   rØ   r'   r   r   rÙ   rÚ   )rÏ   rÑ   rÜ   rH   s       r6   Útensor_funcz%_init_state_dict.<locals>.tensor_funcñ  s{   € Ü˜ ¨$Ó/ˆØ”U—\‘\ &Ó)Ò)Ü×/Ñ/×FÑFÓH×MÑMˆKÜÜ—‘Ô0°Ó=×LÑLÓNóˆFô ×%Ñ% e°FÔ;ˆFØˆMàˆLr8   N)r   r   rK   rº   Ú_iterate_state_dict)r³   rß   râ   rä   s       r6   Ú_init_state_dictræ   Î  s@   € ð
œGó ð*¤3ó ð
œ5Ÿ<™<ó 
ô ØØØØõ	r8   Úiter_objectrß   râ   rä   c           	      ó*  — t        | t        «      r || «      S t        | t        «      r || «      S t        | t        j                  «      r || «      S t        | t
        t        t        t        t        j                  f«      s| €| S t        | t        «      r+| j                  «       D ]  \  }}t        ||||«      | |<   Œ | S t        | t        t        f«      r8| D �cg c]  }t        ||||«      ‘Œ }}t        | t        «      rt        |«      }|S yc c}w )a$  
    Iterate through the state dict, applying the given functions to each tensor type
    and update the state dict in place.

    Args:
        iter_object (Any): the target state_dict.
        sharded_tensor_func (Callable): the function to apply to ShardedTensor
        dtensor_func (Callable): the function to apply to DTensor
        tensor_func (Callable): the function to apply to Tensor

    # TODO: let state_dict_util._iterate_state_dict() to support in place option
    so we don't need to have two versions of _iterate_state_dict.
    N)r·   r   r
   rK   rº   ÚintÚfloatÚstrro   ÚioÚBytesIOÚdictr%   rå   ÚlistÚtuple)rç   rß   râ   rä   rŽ   rÏ   ÚvÚrets           r6   rå   rå     s  € ô( �+œwÔ'Ù˜KÓ(Ð(Ü	�K¤Ô	/Ù" ;Ó/Ð/Ü	�K¤§¡Ô	.Ù˜;Ó'Ð'ä�;¤¤e¬S´%¼¿¹Ð DÔEØÐàÐÜ	�K¤Ô	&Ø%×+Ñ+Ó-ò 	‰JˆC�Ü2Ø�|Ð%8¸+ó ˆK˜Òð	ð ÐÜ	�K¤$¬ Ô	/ð !ö
àô    <Ð1DÀkÕRð
ˆð 
ô �k¤5Ô)Ü˜“*ˆCØˆ
ð 
0ùò
s   ÃD)Erì   r—   Úbisectr   r   Úcollections.abcr   Útypingr   r   rK   Útorch.distributedÚdistributedrÖ   Útorch._utilsr   Ú!torch.distributed._shard.metadatar	   Ú'torch.distributed._shard.sharded_tensorr
   Útorch.distributed.tensorr   Útorch.distributed.tensor._utilsr   r]   r   r   r   r   r   r   r   Úplannerr   r   r   r   r   r   Ú
reshardingr   r   r   rï   rë   Ú__annotations__Úboolr7   rA   rG   rº   rM   rS   r`   ri   rk   rn   rr   r~   r�   r   r¼   rÃ   rÅ   rÉ   rÍ   rî   ræ   rå   r‰   r8   r6   ú<module>r     s‚  ðä 	Û ß 'Ý $ß ã Ý  Ý +Ý ;Ý AÝ ,Ý Q÷÷ ñ ÷÷ ÷ð 9Ð9€ˆˆc‰Ó 9ð>˜hð >°Hð >Àó >ðB	O t¨H¡~ð 	O¸$ó 	OðØ�x‘.ðà�h‘ðð 
ˆ(�^óð2 e§l¡lð Ð7Kó ð˜}ð Ð1Eó ðØ!ðØ-:ðàóð(¨ð °gð À)ó ð(Ø	ðØ+ðØ7Dðàóð
 sð 
°E·L±Lð 
ÀYó 
ð¨ð °Có ò
ò
ðxØ	ðxà(ðxð Ð+Ñ,ðxð 
ˆ(�^ó	xðv°?ð Àxó ð"=˜Sð =¨#ð =°$°y±/ó =ð wð Ð3Gó ð˜uŸ|™|ð °Ð5IÑ0Jó ð&
˜Cð 
 ]ð 
¸ð 
ÀÀhÁó 
ð.4  c¨3 h¡ð 4°Có 4ðn,Øð,àð,ð "ð,ð ô	,r8   