o H&i: @s0UddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZddlZddlmZddlmZddlmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0ddl1m2Z2e3e4Z5ej6e7d<gdZ8GdddeZ9GdddeZ:de e;e fde!defddZdefddZ? d1de ed!e>de e ee!ffd"d#Z@de"de!fd$d%ZAd&ed'ede>fd(d)ZBd*ejCd+ede>fd,d-ZDd.e ede!de>fd/d0ZEdS)2N)ChainMap)reduce)ListTupleDictAnyUnioncast)narrow_tensor_by_index)DTensor) SavePlanner LoadPlannerSavePlanLoadPlanReadItem WriteItem WriteItemType)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadata MetadataIndexMetadataSTATE_DICT_TYPE STORAGE_TYPES)_create_read_items_create_write_items"_create_default_metadata_only_plan)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors) dedup_tensors)find_state_dict_object) set_elementlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc @seZdZUeed<   ddedededdfdd Zd ed eddfd d Zde fddZ de e de e e e ffddZde de fddZdedeejejffddZdedefddZdedefddZdS) r$mappingsTrflatten_sharded_tensorsdedup_replicated_tensorsreturnNcCs||_||_||_i|_dSN)rr+r,r*)selfrr+r,r0WC:\wamp64\www\opt\env\Lib\site-packages\torch/distributed/checkpoint/default_planner.py__init__Gs zDefaultSavePlanner.__init__ state_dictis_coordinatorcCs2|jr t|\}|_|jrt|}||_||_dSr.)rr*r+rr3r4)r/r3r4r0r0r1set_up_plannerRs  z!DefaultSavePlanner.set_up_plannercCs0t|j|j}|jrtj||jd}||_|jS)NZ planner_data)r(r3r4r dataclassesreplacer*plan)r/r9r0r0r1create_local_plan\sz$DefaultSavePlanner.create_local_plan all_planscCsr|jrt|}t|\}}|jr$dd|D}tt|}tj||d}t||s-t d||_ ||_ |j |j fS)NcSsg|]}|jqSr0r6).0pr0r0r1 ssz9DefaultSavePlanner.create_global_plan..r6zFailed to validate global plan) r,r r)rdictrr7r8_validate_global_plan ValueError global_planmetadata)r/r;rBrCZplanner_data_dictZmerged_mappingsr0r0r1create_global_planfs    z%DefaultSavePlanner.create_global_plannew_plancCs ||_|Sr.)r9r/rEr0r0r1 finish_planszDefaultSavePlanner.finish_plan write_itemcC||j}|||Sr.) lookup_objectindextransform_object)r/rHobjectr0r0r1 resolve_datas  zDefaultSavePlanner.resolve_datarKcC t|j|SzSExtension from the planner interface to make it easy to extend the default planner.r!r3r/rKr0r0r1rJ z DefaultSavePlanner.lookup_objectrMcCs(|jtjkrt}t|||}|SrP)typerBYTE_IOioBytesIOtorchsave)r/rHrMbytesr0r0r1rLs  z#DefaultSavePlanner.transform_object)TTT)__name__ __module__ __qualname__r__annotations__boolr2rr5rr:rrrrDrGrrrXTensorrVrWrNrrrJrLr0r0r0r1r$DsF        r$c@seZdZUdZeed<eed<  d&dededdfd d Zd ed e d eddfddZ de fddZ de e de e fddZde de fddZdedejddfddZdefddZdedejddfdd Zd!edejfd"d#Zdedejfd$d%ZdS)'r%z DefaultLoadPlanner that adds multiple features on top of LoadPlanner. In particular it adds the following: flatten_state_dict: Handle state_dict with nested dicts flatten_sharded_tensors: For FSDP in 2D parallel mode original_state_dictr*Trr+r-NcCs||_||_i|_i|_dSr.)rr+rar*)r/rr+r0r0r1r2s zDefaultLoadPlanner.__init__r3rCr4cCs>||_|jr t|}|jrt|\}|_||_||_||_dSr.)rar+rrr*r3rCr4)r/r3rCr4r0r0r1r5s z!DefaultLoadPlanner.set_up_plannercCst|j|jSr.)r&r3rC)r/r0r0r1r:sz$DefaultLoadPlanner.create_local_planrBcCst|Sr.)r')r/rBr0r0r1rDsz%DefaultLoadPlanner.create_global_planrEcCs|Sr.r0rFr0r0r1rGzDefaultLoadPlanner.finish_plan read_itemvaluecCs@|jrt|j|j|jjt|dSt||j|jj<dSr.) rr"rar* dest_indexfqnrXloadr3)r/rcrdr0r0r1 load_bytess zDefaultLoadPlanner.load_bytescCrIr.) lookup_tensorretransform_tensorr/rctensorr0r0r1resolve_tensors  z!DefaultLoadPlanner.resolve_tensorrlcCsdSr.r0rkr0r0r1 commit_tensorrbz DefaultLoadPlanner.commit_tensorrKcCrOrPrQrRr0r0r1rirSz DefaultLoadPlanner.lookup_tensorcCst||j|jSrP)r Z dest_offsetslengthsrkr0r0r1rjs z#DefaultLoadPlanner.transform_tensor)TT)r[r\r]__doc__rr^rr_r2rr5rr:rrDrGrrVrWrhrmrXr`rnrrirjr0r0r0r1r%s<      r%r3rCr-cCsdg} |D]&\}}|j|}t|tr%|jdur$|t|||7}q|t|||7}qt|Sr.)itemsstate_dict_metadata isinstancer device_meshget_coordinaterr)r3rCrequestsrfobjmdr0r0r1r&s  r&r;cCs|S)z Create global load plan used by DefaultLoadPlanner. The default load behavior involved no global coordination and this function currently doesn't change the local plans. r0)r;r0r0r1r's r'r4cCsdg}|D]'\}}t|tr|jdur|t||7}qt|tjs&|r-|t||7}qt|S)a Create the ``SavePlan`` used by DefaultSavePlanner. On non-coordinator ranks, this function ignores tensors and non-tensor objects, only producing writes for ShardedTensor objects. On the coordinator rank, produce writes for all values. N) rqrsr rtrurrXr`r)r3r4rvrfrwr0r0r1r(s  r(Trewrite_index_hintsc Cs i}g}|D]}g}|jD]q}|jtjks|jj|vsJ|jtjkr0t||jj<||q |j dus7Jt t | |jjt |j j |j jgd}|}|ratj|jt|jd} tj|| d}|||j jdusvJd|jjd|j|j jq |tj||dq|t|fS)a6 Create the global plan and metadata used by DefaultSavePlanner. Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans. The only global planning change is to update index hints in all ``MetadataIndex`` objects if ``rewrite_index_hints`` is True. N) propertiessizechunks)rKzZ Cannot create MD for tensor without bounds. FQN: z )rq)rqrTrZSHARDrKrfrUrappendZ tensor_datar r setdefaultrzr{r7r8lenr|chunkr) r;ryrxZ new_plansr9Z new_itemsitemZ tensor_mdZnew_itemZ new_indexr0r0r1r)&sJ         r)cCst|}t|g\}}|S)zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)rr))r3r9_rxr0r0r1_create_default_local_metadata]srbox0box1cCsdt|j}t|D]&}|j||j||j|krdS|j||j||j|kr/dSq dS)z9Check if two boxes overlap. Tuples are (offset, lengths).FT)roffsetsrangesizes)rrZndimsir0r0r1_check_box_overlapds  router_box_size inner_boxcCs`tt|D]'}|j|dkrdS|j|dkrdS|j||j|||kr-dSqdS)NrFT)rrrr)rrrr0r0r1_check_box_boundsvsrrBc Csd}|jD]j\}}t|trqt|jdkrqd}t|jD]:\}}t|j|s5t d||j|d}|t t j |jd7}|j|ddD]}t||rYt d|||d}qHq t t j |jd} || krqt d|| |d}q|S)NTrz~ key:%s has out of bounds chunk: tensor-size:%s chunk: %s Fz$key:%s has overlapping chunks: %s %szq key:%s invalid fill tensor-volume: %s chunks-volume: %s )rrrqrsrrr{ enumerater|rr#warningroperatormulrr) rBrCall_goodkeyrdZ chunks_volumeZ chunk_idxZchunk0Zchunk1Z tensor_volumer0r0r1r@s@   r@)T)Fr7rVloggingr collectionsr functoolsrtypingrrrrrr rXZtorch.distributed._shard._utilsr Ztorch.distributed._tensorr Z$torch.distributed.checkpoint.plannerr r rrrrrZ%torch.distributed.checkpoint.metadatarrrrrrrZ,torch.distributed.checkpoint.planner_helpersrrrZ)torch.distributed.checkpoint._nested_dictrrZ2torch.distributed.checkpoint._sharded_tensor_utilsrZ+torch.distributed.checkpoint._dedup_tensorsr Z"torch.distributed.checkpoint.utilsr!Z&torch.distributed.checkpoint._traverser" getLoggerr[r#Loggerr^__all__r$r%strr&r'r_r(r)rrSizerr@r0r0r0r1s      $$       TN      7