o
    H&iQ                     @   s   d dl Z d dlmZmZmZmZmZmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ ddgZe
eZed	d
G dd dZed	d
		ddedejjdeegef deeeef  dee f
ddZdS )    N)AnyCallableDictListOptionalSet)OrderedDict)compatibility)GraphModule)Node	Partitionsplit_moduleT)Zis_backward_compatiblec                   @   s(   e Zd ZdefddZdefddZdS )r   namec                 C   sN   || _ d| | _g | _i | _i | _i | _i | _tjj	
 | _	i | _i | _d S )NZsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr    r   GC:\wamp64\www\opt\env\Lib\site-packages\torch/fx/passes/split_module.py__init__   s   
zPartition.__init__returnc                 C   s4   d| j  d| j d| j d| j d| j d| j S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r   r   r   r   __repr__   s   
zPartition.__repr__N)__name__
__module____qualname__strr   r    r   r   r   r   r      s    Fmroot_msplit_callbackqualname_mapkeep_original_orderc           0   
      s  dt dttt f dtttjjjf ffdd}i i dt dtt  ffdd		
fd
d}tjj	tjj
tjjg}t }t }	i }
d}t }jjD ]jdv rUqM| jdkrڈj|v rڈjtjjkrtjdkssJ tjd ts}J }t
h||< nQjtjj	krtdd jD sJ | t
h|	< d|
< n+jtjj
krtjdksJ |	jd  
 |jd  |
jd < |dur|| 
 |D ]}|	| 
 qqMtdd |
 D sJ ddd |	 D }	dd | D }ttjr+t d|	 t d| t|	p3t|}d}jjD ]aj!< jdv rIq:jdkr`tjj"jd 	fdd q:|rx
}||ksvJ d| d | |}j|vrtjj"j	fd!d tjj"j#	fd"d q:t$% }g } D ]\}tj&s|'| qg }|r|( }|'| | j)D ]}| j&(| | j&s|'| q|st|tkrt*d#|	|fD ]R}| D ]J\}t|dks
J t|d  j+< |dd D ]*}t| jj,jjt-d$d jD i j.d%}j/0 |_/|j+< qqq|D ](}| j1D ]}jj2|| j.d&}| j/0 |_/|j+| < qUqLjjD ]t3d'rj4 j+tjj"jfd(d}tjj"j#fd)d}jd*vrj}n>j5d+} }!| D ]}"t3|!|"st6d,j d-t7|!|"}!qd.8| }|!j9|< |durj: d+| }#j||#< t|t-sJ t|t;sJ jj,j|||j.d%}j/0 |_/|j+< qy|	fD ]M}t<|D ]E| }t|dks-J |dd D ].}t| |
 }$|$dusHJ d/jj,|$j|$jj+ fi |$j.d%}|$j/0 |_/q3qqi i  tjj= i }%|sjjD ]| |%\ }%qynjjD ]j!< q|s|n|}&t }'|&D ]}| t-fd0dj>D }(t|(})|)dkrĈj?|(d  n|)dkrψj?|( |rfd1d2j1D }*|*D ]|'v rq| |%\ }%|' qtjjj9j|%j:< @j:t- fd3dj1D }+tj>},|,dkr9tjjAB|+}-tCj>D ]\}.}/|-|. jD |/< q*q|,dkrG|+ tEtFj>< qjjD ]jdkrg?tjj"jd  fd4d qMtjj|%S )5a  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule


    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self):
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    nodebase_mod_envbase_mod_attrsc                    s   | j dkr/t| jdkr| jd ntjj} j| j| j|d|| j	< | j
 || j	 _
||fS | j dkri | j|| j	< | j
 || j	 _
}| jdD ]}t||s^td| j dt||}qN||| j< ||fS )Nplaceholderr   )	type_exprdefault_valueget_attr.zNode target  not found!)oplenargsinspect	Signatureemptyr-   targettyper   metacopyr0   splithasattrAttributeErrorgetattr)r*   r+   r,   r/   Zattr_valatom)base_mod_graphr%   r   r   construct_graph   s$   





z%split_module.<locals>.construct_graphdef_nodeuse_nodec                    s   t | dd }t |dd }||krD|d ur) | }|j| j |d ur)|j| |d urF | }|j| j |d urH|j| d S d S d S d S )N_fx_partition)r@   r   
setdefaultr   r   r   r   )rD   rE   ZdefinedusedZdef_partitionZuse_partition)
partitionsr   r   record_cross_partition_use   s    z0split_module.<locals>.record_cross_partition_usec                    sF   t | } |}|d u rt|  |< }|j| j || _d S N)r$   getr   r   appendr   rF   )r*   partition_name	partition)rI   r'   r   r   "instantiate_node_partition_mapping   s   

z8split_module.<locals>.instantiate_node_partition_mappingN)r-   r0   outputcall_function   r   c                 s   s    | ]	}t |t V  qd S rK   )
isinstancer   .0argr   r   r   	<genexpr>   s    zsplit_module.<locals>.<genexpr>c                 s   s    | ]}|d uV  qd S rK   r   )rV   vr   r   r   rX          zautocast must exitc                 S      i | ]	\}}|t |qS r   sortedrV   krY   r   r   r   
<dictcomp>      z split_module.<locals>.<dictcomp>c                 S   r[   r   r\   r^   r   r   r   r`     ra   zautocast_regions: %szgrad_regions: %s)r-   r0   rQ   c                    s
    | d S rK   r   n)rJ   r   r   <lambda>     
 zsplit_module.<locals>.<lambda>zRautocast or set_grad_enabled require monotonically increasing partitions:highest: z, this node's: c                    
   |  S rK   r   rD   r*   rJ   r   r   re   %  rf   c                    rg   rK   r   rh   ri   r   r   re   (  rf   z cycle exists between partitions!c                 s   s    | ]}|V  qd S rK   r   rU   r   r   r   rX   H  s    )r3   r9   r5   kwargsr.   )r.   rF   c                        |  S rK   r   rc   r   r   r   re   a      c                    rk   rK   r   rc   rl   r   r   re   c  rm   )call_moduler0   r1   zOperator target r2   _zMissing exit nodec                 3   s    | ]
}j  |  V  qd S rK   rl   rV   r   )
orig_nodesrO   r   r   rX     s    
c                    s   g | ]} | qS r   r   )rV   key)orig_mod_envr   r   
<listcomp>  s    z split_module.<locals>.<listcomp>c                 3   s    | ]} | V  qd S rK   r   rp   r+   r   r   rX     rZ   c                    s
    | j  S rK   )r   rc   ru   r   r   re     rf   )Gr   r   r$   r   r   Zgraph_moduler
   r   ampZ_enter_autocastZ_exit_autocastZ_CZ_set_grad_enabledr   setr   nodesr3   r9   r4   r5   rT   boolalladdremovevaluesitems_LOGGERisEnabledForloggingDEBUGdebugr   Zmap_argrj   listkeysr   rM   popr   RuntimeErrorr   Zcreate_nodetupler:   r;   r<   r   r-   r>   rF   r=   r?   r@   joinr   r   dictreversedr   r   rQ   rn   proxyZProxy	enumerater*   nextiter)0r%   r&   r'   r(   r)   rC   rP   ZGLOBAL_STATE_NODESZgrad_regionsZautocast_regionsZautocast_exitsZactive_gradZactive_autocastsaZassert_monotonically_increasingZhighest_partitionpidZoriginal_partition_orderZroot_partitionsrN   Zsorted_partitionsZroot_partitionZ	dependentZregions_mappingZregionsrnew_nodeinpr-   Zgathered_argsZgathered_kwargsr9   Ztarget_atomsZtarget_attrrA   qualnameZ	exit_noder,   Zconstruct_order_partitionsZalready_constructed_attr_nodesZoutput_valsZnum_output_valsZorig_mod_attr_nodesZ
output_valZnum_outputsZoutput_val_proxyiZoutput_namer   )r+   rB   r   r%   r*   rs   rq   rO   rI   rJ   r'   r   r   (   s  g

	


 




	







	







)NF)r6   typingr   r   r   r   r   r   collectionsr   r   r   Ztorch.fx._compatibilityr	   Ztorch.fx.graph_moduler
   Ztorch.fx.noder   __all__	getLoggerr!   r   r   nnModuleintr$   ry   r   r   r   r   r   <module>   s4     
