o
    H&i"                  	   @   s  U d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZmZ ddlmZmZ dgZee  e!d< dej"defddZ#dedefddZ$dededefddZ%de dedefddZ&de dededefddZ'de dej"defddZ(de defd d!Z)d"d# Z*d$d% Z+de d&ed'ee dee fd(dZ,d)edefd*d+Z-de d,edee fd-d.Z.dedefd/d0Z/de d1ed2edee fd3d4Z0dS )5    )AnyListN)ShardMetadata)ShardedTensor)TensorProperties)DTensor)%compute_local_shape_and_global_offset   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__tensorreturnc                 C   s$   t tdgt|   |  dS )Nr   offsetssizes)r   torchSizelensize)r    r#   WC:\wamp64\www\opt\env\Lib\site-packages\torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensor%   s   r%   shard_mdc                 C   s   t t| jt| jdS Nr   )r   r   r    shard_offsetsZshard_sizes)r&   r#   r#   r$   _chunk_for_shard+   s   

r)   sharded_tensorc                 C   s   t t||  j|  jdS )Nchunk
propertiesr"   )r   r)   metadataZtensor_propertiesr"   )r*   r&   r#   r#   r$   _sharded_tensor_metadata2   s
   r/   fqnc              	   C   sb   t |j|j|j\}}t|t|}}tt| |tj	t
t||dt| | ddS )Nr   r+   indextypeZtensor_data)r   shapedevice_mesh
placementsr   r    r   r   r   SHARDr   r   r   create_from_tensorZto_localr"   )r0   r   r   r   r#   r#   r$   _create_write_items_for_dtensor<   s    r9   c                 C   s(   t |j}tt| |tjt||dS )Nr1   )r   r    r(   r   r   r   r7   r/   )r0   r*   r&   r   r#   r#   r$   _create_write_item_for_shardQ   s   r:   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r   r+   r1   )r   r    r!   r"   r   r   r   TENSORr   r   r   r8   )r0   r   r   r#   r#   r$   _create_write_item_for_tensor\   s   r<   bytesc                 C   s   t t| tjdS )N)r2   r3   )r   r   r   BYTE_IO)r0   r=   r#   r#   r$   _create_write_item_for_bytesioi   s   r?   c              	   C   s.   t tj| t|f|t|ft|fdS N)r3   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   r>   r   r    rA   Zdest_offsetrC   Zstorage_offsetlengthr#   r#   r$   _create_read_item_for_byteiop   s   


rH   c              	   C   s(   t tj| t||t|t|dS r@   )r   r   r;   r   r    rA   rB   rC   rD   rE   r#   r#   r$   _create_read_item_for_tensor}   s   rJ   checkpoint_mdlocal_chunksc                 C   s   g }t |D ]L\}}t |jD ]B\}}t||sqg }g }	g }
t||dD ]\}}}}|| |	| |
| q%|tt| |j||	t| |j|||
d qq|S )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )Zsaved_shardZcurrent_shardrI   )	enumeratechunksr   r   appendrJ   r   r   )r0   rK   rL   Z
read_itemsidxshardZstorage_idxZ
storage_mdrD   rB   rE   dimZoffset_for_saved_tensorZoffset_for_current_tensorrG   r#   r#   r$   r      s<   



state_dictc                 C   s   g }|   D ]@\}}t|tr|t|| qt|tr/| jD ]}|t||| q"qt|t	j
r>|t|| q|t|| qt|S )N)items
isinstancer   rO   r9   r   r.   Zshards_metadatar:   r   Tensorr<   r?   r   )rS   requestsr0   objr&   r#   r#   r$   "_create_default_metadata_only_plan   s   

rY   objectc                    s\   t trt gS t tr fdd D S t tjr(t gS t gS )Nc                    s   g | ]	}t  |jqS r#   )r:   r.   .0rQ   r0   rZ   r#   r$   
<listcomp>   s    z'_create_write_items.<locals>.<listcomp>)	rU   r   r9   r   local_shardsr   rV   r<   r?   r]   r#   r]   r$   _create_write_items   s   

r`   c                 C   s8   t | j| j| j\}}t|t|}}t||dS r'   )r   r4   r5   r6   r   r    r   )r   r   r   r#   r#   r$   _create_chunk_from_dtensor   s   ra   mdrX   c                 C   s   t |ts@t |trt|g}n*t |trdd | D }nt |tjr+t|g}nt	d|  ddt
|  t| ||S tt| dt| dddgS )Nc                 S   s   g | ]}t |jqS r#   )r)   r.   r[   r#   r#   r$   r^      s    
z&_create_read_items.<locals>.<listcomp>z Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rF   )rU   r
   r   ra   r   r_   r   rV   r%   
ValueErrorr3   r   rH   r   )r0   rb   rX   rL   r#   r#   r$   _create_read_items   s.   



rd   )1typingr   r   r   Z!torch.distributed._shard.metadatar   Z'torch.distributed._shard.sharded_tensorr   Z0torch.distributed._shard.sharded_tensor.metadatar   Ztorch.distributed._tensorr   Z torch.distributed._tensor._utilsr   r.   r
   r   r   r   r   r   Zplannerr   r   r   r   r   r   Z
reshardingr   r   r   str__annotations__rV   r%   r)   r/   r9   r:   r<   r?   rH   rJ   r   rY   r`   ra   rd   r#   r#   r#   r$   <module>   s^      		



7"