o
    I&iq4                    @   sn  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ zd dlZW n eyM   dZY nw d dlZd dlmZ d dlmZmZ d dlmZ d dlmZmZ d dlmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d d	l)m*Z* d d
l+m,Z, d dl-m.Z. d dl/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@ ddlAmBZBmCZCmDZD ddlEmFZF ddlGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZR ddlSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZc ddldmeZemfZfmgZg ddlhmiZi ddljmkZkmlZl ddlmmnZnmoZompZpmqZq ddlrmsZsmtZtmuZumvZvmwZwmxZx ddlymzZzm{Z{m|Z|m}Z} ddl~mZmZmZmZmZ ddlmZ ddlmZ dd lmZmZmZmZmZmZmZmZmZ dd!lmZmZmZmZmZmZmZmZmZmZmZmZmZ dd"lmZmZ dd#lmZ dd$lmZmZmZmZmZ dd%lmZmZ dd&lmZmZ dd'lmZmZmZ eeZeZG d(d) d)ZejG d*d+ d+ZejG d,d- d-ZG d.d/ d/Zd0d1 ZdAd2d3Z	dAd4d5ZejG d6d7 d7Zd8e(fd9d:Zd;eeQ d<efd=d>ZG d?d@ d@ZdS )B    N)List
NamedTupleOptionalUnion)SymInt)GuardSourceTracingContext)HigherOrderOperator)
_EventBase_StreamBase)
FakeTensoris_fakemaybe_get_fake_mode)_constrain_range_for_sizeDimConstraint
DimDynamicRelaxedUnspecConstraintStatefulSymbolicContextSymbolicContext)immutable_list)NestedTensor)is_traceable_wrapper_subclass)TensorWeakRef   )configmutation_guardreplay_record	skipfilestrace_rules)
is_allowedis_builtin_callableis_numpyis_user_defined_allowed) get_registered_device_interfaces)InternalTorchDynamoErrorunimplemented)GuardBuilderinstall_guardmake_dupe_guard)SideEffects)
AttrSourceConstantSourceConvertIntSourceGetItemSourceGlobalWeakRefSourceis_constant_sourceLocalSourceNumpyTensorSourceRandomValueSourceSourceTupleIteratorGetItemSource)build_checkpoint_variableclone_inputget_fake_valueget_static_address_typeglobal_key_nameis_namedtuple	is_typingis_utils_checkpointistypeodict_valuespreserve_rng_statetensor_always_has_static_shapetuple_iteratortuple_iterator_getitemtuple_iterator_lenwrap_fake_exception   )MutableLocaltypestrVariableTracker)BuiltinVariable)ConstantVariableEnumVariable)AutocastModeVariableEventVariableNullContextVariableStreamVariable)ConstDictVariableDataClassVariableDefaultDictVariableHFPretrainedConfigVariablePythonSysModulesVariableSetVariable)DeviceMeshVariablePlacementClassVariablePlacementVariableProcessGroupVariable)!CollectiveFunctionRewriteVariableFunctoolsPartialVariableTritonKernelVariableUserFunctionVariableUserMethodVariable) TorchHigherOrderOperatorVariable)LazyVariableTracker)	BaseListVariableListVariableNamedTupleVariableRangeVariableRestrictedListSubclassVariableSizeVariableSliceVariableTupleIteratorVariableTupleVariable)AutogradFunctionContextVariableAutogradFunctionVariableComptimeVariableGetAttrVariableGetSetDescriptorVariableInspectSignatureVariableLambdaVariableMethodWrapperVariableNumpyVariablePythonModuleVariableSavedTensorBoxSkipFilesVariableTypingVariable)FSDPManagedNNModuleVariableUnspecializedNNModuleVariable)OptimizerVariable)NumpyNdarrayVariableSymNodeVariableTensorSubclassVariableTensorVariableUnspecializedPythonVariable)torch_special_class_typesTorchVariable)build_torch_function_fnTensorWithTFOverrideVariable)KeyedJaggedTensorVariableUserDefinedClassVariableUserDefinedObjectVariablec                   @      e Zd ZdS )_missingN__name__
__module____qualname__ r   r   JC:\wamp64\www\opt\env\Lib\site-packages\torch/_dynamo/variables/builder.pyr          r   c                   @   s   e Zd ZU eed< eeejf ed< e	ed< e
ejjj ed< dZe	ed< dZe
ej ed< ed	d
 Zdd Zdd Zdd Zdd ZdS )GraphArgsource_exampleis_unspecializedfake_tensorT	is_tensorNexample_strong_refc                 C   s*   t | jtr|  }|d usJ |S | jS N)
isinstancer   r   )selfrr   r   r   example   s
   zGraphArg.examplec                 C   s0   t | jtjrt| j| _t| jsJ d S d S r   )r   r   torchTensorr   r   r   r   r   r   r   __post_init__   s
   zGraphArg.__post_init__c                 C   s   | j |S r   )r   Zreconstructr   txr   r   r   load      zGraphArg.loadc                 C   s   d | _ d | _d S r   )r   r   r   r   r   r   erase   s   
zGraphArg.erasec                 C   s   | j  |j  kS r   )r   namer   otherr   r   r   __eq__      zGraphArg.__eq__)r   r   r   r3   __annotations__r   r   r   r   boolr   _subclassesr   r   r   r   r   propertyr   r   r   r   r   r   r   r   r   r      s   
 
r   c                   @   s*   e Zd ZU ee ed< eee  ed< dS )FrameStateSizeEntryscalarsizeN)r   r   r   r   intr   r   r   r   r   r   r      s   
 r   c                       s0  e Zd ZdZdef fddZdd Zdd Zee	
d	d
d Zdd Zdd Zdd Zee	
d	dd Zee	
d	dd Zdd Zdd Zdd Zdeeeeef fddZdefdd Zdeeef fd!d"Zde j!j"fd#d$Z#d%d& Z$de j%fd'd(Z&de j%fd)d*Z'd+d, Z(d-d. Z)  Z*S )/VariableBuilderz3Wrap a python value in a VariableTracker() instancer   c                    sH   |d usJ dt  d usJ dt   || _|| _| | _d S )NzRConsider SourcelessBuilder for ephemeral objects, usually objects created locally.zExpected active TracingContext)r   Ztry_getsuper__init__r   r   r   )r   r   r   	__class__r   r   r      s   

zVariableBuilder.__init__c                 C   sz   || j jjv r| j jj| }t| j|j}|r| | |S | |jdi |  }| 	|r;| j jj
| j||}|S )Nr   )r   outputside_effectsr(   r   install_guards_wrapcloneoptions_can_lift_attrs_to_inputstrack_object_existing)r   valueZside_effect_resultZ	dup_guardvtr   r   r   __call__   s   


zVariableBuilder.__call__c                 C   s   t |ttttfv rdS dS )NTF)typer}   r   r   rz   )r   r   r   r   r   r      s   z)VariableBuilder._can_lift_attrs_to_inputsNc                   C   s   ddhS )Nr   rE   r   r   r   r   r   _common_constants  s   z!VariableBuilder._common_constantsc                 C   s   | j S r   r   r   r   r   r   
get_source  s   zVariableBuilder.get_sourcec                 C   s   d|   iS Nr   )r   r   r   r   r   r     r   zVariableBuilder.optionsc                    sD   |    t ts  tjkrd S t fdd|D ddi i S )Nc                    s   g | ]}  |qS r   )
make_guard).0guardr   r   r   
<listcomp>!      z2VariableBuilder.install_guards.<locals>.<listcomp>skiprE   )r   r   r+   guard_sourcer   ZCONSTANTr'   )r   guardsr   r   r   r     s   zVariableBuilder.install_guardsc                 C   s   t jt jjt jjt jjjf| jft	t
ttjf| jft| jfttf| jfttttd tt jt jt jf| jfg}tjrDtrD| tj!| j"f i }|D ]\}}t#|t	rS|n|fD ]}||vs^J |||< qVqH|S r   )$r   r   nn	Parameterr   r   functional_tensorFunctionalTensorwrap_tensortuplelistr>   collectionsdequewrap_listlikerA   wrap_tuple_iteratorslicerangewrap_slice_ranger   floatr   r   strSizedeviceZdtypewrap_literalr   Ztrace_numpynpappendndarraywrap_numpy_ndarrayr   )clsentriesresulttsfntr   r   r   _type_dispatch$  s>   	


zVariableBuilder._type_dispatchc                 C   s~   ddl m } tjdd f|dd ftjdd fg}i }|D ]\}}t|ttfr*|n|fD ]}||vs5J ||t|< q-q|S )Nr   )comptimec                 S   s    t tjfd| ji| tjS r   )rp   ro   creater   r   r&   CLOSURE_MATCHr   r   r   r   r   <lambda>W  s    
z.VariableBuilder._id_dispatch.<locals>.<lambda>c                 S   s   t  S r   )rl   r   r   r   r   r   ]  s    c                 S   s   t tfd| ji| tjS r   )rp   _dataclasses_fields_lambdar   r   r&   FUNCTION_MATCHr   r   r   r   r   `  s    
)	r   inspect	signaturedataclassesfieldsr   r   r   id)r   r   r   r   r   r   r   r   r   r   _id_dispatchO  s    
zVariableBuilder._id_dispatchc              
      s  ddl m} | rddlm} ddlm} nG dd d}G dd d} t}|d ur6|S 	 t
}|d urH|S td	tsYttjr^S trgS tjjju rtj fd
d D }t|tS tju rtj dS tt!t"j#t"j$frt%fdd D rs& ' rtj( ntj)  D ]}*|rԈj+,t-|| qćfdd  fdd D }tt"j#rt.|t/j0}nt|t}j+j1j23j |S t4tj5j6r7S t89r%:S tt;rCt%dd D rCtj< t8j=j dS t4t>j?rWtj< t@j dS tAritjB tCj dS tDrttEj dS t4tFjGrtH& d}	tIj+|	jJ}
g }tH& d}tKjLD ]\}}|MtIj+tN||| qi }tH& d}jOP D ]\}}tIj+tN|||||< qtQ& RtjS|Rtj)|RtjT tU|
||dS tVrtj< tWj dS tXd urt4tXjYrZtX[S t\r/tXsJ t]r$tj^ntjS t_j dS t`arEtj^ t`j=j+j dS ttjbjcjdr[tj^ tej dS t4tjbjcjfrtHj dtQj RtjSRtjT fddtKjgD }j+j1j2hj tij tj|dS t4tkjlrttmdd tjbjcjdrtmdddkrЈtmjndd krЈtj^ totejnj ddS tXrt4tXjpr߈qS trsrtjS trtS tusrtjS tuS t4tvrtjStjw txjyj dS tjzd kr0t4jnt{r0tj^ t|S t4t}rGtj< t~d jjj dS t4tr[tj< td j dS t4tjjrptjv rptj dS ttjrtd!d d u rtjS tj dS tsrtjS tj d}j+j1j2hj |S t4tjjrtjS tj dS trӈtj< tj dS trtj< tj dS trtj< tj dS trtj< tj dS t4tjrij }tj }j+j1jjt||tjd"}j+j1jjtd#d$jt||d}t||d%d d%|d&|jjd'< j+j1j|jj j+j1jMt||d  t||d(kS t4||frtj< td d j dS t4tjjjrtj< tjjjjgj d)S td urtrd*j+j1_tjj dS tr͈tj^ t|j dS tttkjfrtjd*d+rtd	d%std,d%stj^ ttjd*d+jj dS ttkjtjjfrtj tj dS t4tkjlrTt4jntj5j6rTtIj+tHj ddjn}|rAt4|tsEJ d-tj^ tÈj|j dS ttkjtjfrktj^ tȈj dS t4tkjɃr|tj^ tʈS t4tkj˃rtj^ ẗj dS tttrtj^ tΈj dS tϠtr׈tjStjT j+j1j2шj tχfd.dttӈD ttHj d/d0S tjS tԈj d}tՠts|S j+j1j2hj |S )1Nr   )
has_triton)	Autotuner)JITFunctionc                   @   r   )z*VariableBuilder._wrap.<locals>.JITFunctionNr   r   r   r   r   r   y  r   r   c                   @   r   )z(VariableBuilder._wrap.<locals>.AutotunerNr   r   r   r   r   r   |  r   r   Z_torchdynamo_inlinec                    s(   i | ]}|t | t  |d qS r   )r   r-   r   r   kr   r   r   
<dictcomp>  s    z)VariableBuilder._wrap.<locals>.<dictcomp>r   c                 3   s0    | ]}t |p |pt|tjV  qd S r   )rJ   
is_literaltensor_can_be_dict_keyr   enumEnumr   r   r   r   	<genexpr>  s    


z(VariableBuilder._wrap.<locals>.<genexpr>c                    s     | rtt| S | S r   )r  r.   r9   )keyr   r   r   index_source  s   
z+VariableBuilder._wrap.<locals>.index_sourcec              	      s.   i | ]}|t j| t  |d qS r   )r`   r   r-   r   r   )r  r   r   r   r   r     s    c                 s   s"    | ]}t |pt|V  qd S r   )r   rJ   r   r   xr   r   r   r    s     r   r   funcargskeywords)originalsaved_tensorsc                    s&   g | ]\}}t jt ||qS r   )r   r   r-   )r   nv)saved_tensors_sourcer   r   r   r   %  s    z)VariableBuilder._wrap.<locals>.<listcomp>)r   r  __self__r    applyZbuiltin_function_or_methodenter_result)dynamic_dim[^a-zA-Z0-9]+_Fr   r   graphargrE   )Ztarget_valuesr   T)Zis_inlined_callZ__script_if_tracing_wrapperz"Failed to produce a valid self objc                    s&   g | ]}t j| t j|d qS )r	  )r`   r   r-   r   r   ir   r   r   r     s    r   )user_clsZuser_cls_source)Ztorch.utils._tritonr   Ztriton.runtime.autotunerr   Ztriton.runtime.jitr   r   getr   r   r   r   getattr_staticr   r=   r   traceable_tensor_subclassesr   r:   r   r   utilsZ_pytreeZSUPPORTED_NODESr   r&   ZDICT_VERSIONkeysrP   sysmodulesrT   r   dictr   defaultdictOrderedDictallr   is_nn_moduleZ
BOOL_FALSEZ	DICT_KEYSr  r   Zstore_global_weakrefr9   rR   r   default_factoryr   r   Z
track_dictr   r   Modulewrap_modulerJ   r   r   	frozensetID_MATCHr   r  r  rK   r    ZBUILTIN_MATCHrI   r<   r5   	functoolspartialr*   r   r
  	enumerater  r   r-   r  itemsr'   r   
TYPE_MATCHLIST_LENGTHr[   r;   rv   r   Zgenericr   Zasarrayr!   callabler   rr   rZ   Zcan_rewriteZautogradfunctionZFunctionMetark   ZFunctionCtxr  r   rj   rt   types
MethodTypegetattrr  rm   numberwrap_unspecialized_primitiverQ   Zis_matching_objectwraprS   r	   Z
NAME_MATCHr_   maker   r   r   r   rO   r   r
   rM   _CZ_TensorMetar|   
contextlibnullcontextrN   r   optim	Optimizerry   rY   Zis_process_grouprV   Zis_device_meshrW   Zis_placement_typerX   Zis_placementSymBoolnodeZrequire_hintr,   	shape_env$create_unspecified_symint_and_symbolr   r   DYNAMICroot_tracercreate_graph_inputresubr   r   metabound_symbolsaddexprtracked_fakesTrackedFaker{   r\   ampZautocast_modeZautocastrL   Z
fast_dtypeZ_enabledZ_cache_enabledr   lookupr"   !has_user_defined_allowed_in_graphZcreate_with_sourcer   FunctionTyper   checkru   Zcheck_verbosereasonZjitZScriptFunctionr   r]   rH   r^   __func__
ModuleTyper   ZDummyModulers   GetSetDescriptorTypern   MethodWrapperTyperq   
issubclassr   re   Zis_matching_cls
track_listr   lenr   r)   "cls_supports_mutation_side_effects)r   r   r   r   r   Ztype_dispatchZid_dispatchr   r  Zfunc_srcZfunc_objr  Zargs_sourcer  argr  Zkeywords_sourcer   r  r  Z
value_hintZ
new_sourceZ
new_symintZsym_node_proxyZself_objr   )r  r  r   r   r   r   p  s  































	








zVariableBuilder._wrapc                 C   s4   t |tjjpt | jto| jjdkot | jjtS )Nstate)	r   r   r   r   r   r*   memberbaser0   r   r   r   r   r  (  s   
z&VariableBuilder.tensor_can_be_dict_keyc                 C   s   | j o[t| j to[t| j jto[| j jjdko[t| j jjto[t| j jjjto[| j jjjjdko[t| j jjjjto[| j jjjjj| j	j
 v rZt| j	j
| j jjjjj tjjS dS )NparamsZparam_groupsT)r   r   r-   rc  indexr*   rb  r0   Z
local_namer   f_localsr"  r   rA  rB  r   r   r   r   tensor_should_specialize1  s.   
z(VariableBuilder.tensor_should_specializer   c                    st     tj |D ]
}||u rtd q fddt|D }t||t d}t|t	r8 j
jj j||S |S )Nz-list elements are pointing to the list itselfc                    *   g | ]\}}t  jt  ||qS r   r   r   r-   r   )r   r  itemr   r   r   r   N      z1VariableBuilder.wrap_listlike.<locals>.<listcomp>mutable_local)r   r&   r4  r%   r1  ra   Zcls_for_instancerF   r=   r   r   r   r   r]  r   )r   r   rj  r   r   r   r   r   r   E  s   

zVariableBuilder.wrap_listlikec                    s6     tj  fddttD }t|t dS )Nc                    ,   g | ]}t  jt  |t|qS r   )r   r   r4   r   rB   r  r   r   r   r   [      z7VariableBuilder.wrap_tuple_iterator.<locals>.<listcomp>rl  )r   r&   ZTUPLE_ITERATOR_LENr   rC   rh   rF   )r   r   r   r   r   r   r   Y  s
   
z#VariableBuilder.wrap_tuple_iteratorc                    s:    fdddD }  tj ttrt|S t|S )Nc                    rn  r   )r   r   r*   r   r9  r   r   r   r   r   d  ro  z4VariableBuilder.wrap_slice_range.<locals>.<listcomp>)startstopstep)r   r&   r3  r   r   rg   rd   )r   r   r2  r   r   r   r   c  s   
z VariableBuilder.wrap_slice_rangec                 C   s.  ddl m} t||r| tj t| jd| _| |j	S t
|tjjtjjtjjfr3tjs3td t|rV| tj t|}tt|sK|S | jjj| j||S t|jtjjj j!rj| tj t|S t"|ddrt"|ddszJ d| tjtj# t$|| % d	S | jjj&|| j'| % d	S )
Nr   )OptimizedModule	_orig_modz5TorchDynamo purposely graph breaks on RNN, GRU, LSTMsZ_is_fsdp_managed_moduleFZ_fsdp_use_orig_paramsz3Dynamo only supports FSDP with use_orig_params=Truer   )(Z
eval_framers  r=   r   r&   r3  r*   r   r,  rt  r   r   r   ZRNNZGRUZLSTMr   Z	allow_rnnr%   r   Zis_dynamic_nn_modulerx   r)   r_  r   r   r   r   r   r\  r   ZparalleldistributedZDistributedDataParallelr9  r.  rw   r   register_attr_or_moduler   )r   r   rs  r   r   r   r   r,  p  sH   


zVariableBuilder.wrap_modulec                    s   t j }|r t|tju r  tj t fddt	|D S |rRt|t
u rRt jsM|  v sA j  rA j  rM tj tj|dS  |S  tj tj|dS )Nc                    rh  r   ri  )r   r  r  r   r   r   r     rk  z0VariableBuilder.wrap_literal.<locals>.<listcomp>r   )r   specialize_intr   r   r   r   r&   r4  rf   r1  r   r   r  #force_unspec_int_unbacked_size_liker   r   r   is_localr)  CONSTANT_MATCHrJ   r   r;  )r   r   Zunspecr   r   r   r     s(   


zVariableBuilder.wrap_literalc                 C   s*   t |rt|| jju rtddd S d S )Nz*Cannot wrap a Tensor that has already beenz"wrapped by this instance of Dynamo)r   r   r   	fake_moder$   r   r   r   r    assert_not_wrapped_by_this_graph  s   z0VariableBuilder.assert_not_wrapped_by_this_graphc           
   	   C   sZ  |   }|| jjjvsJ |  st|d ur/|  s/| | | jjj	|| j
|dS t|rH| | | jjj	|tdd| j
|dS t|tjv rTt|}nt|tjtjjtjjjtjjjfv sqt|sqJ t|d }|| jjjv }|r| jjj| S | | | jjjjtdd| j
t||d}i }t|tjv rt| j|| j|d< |  t!j" t#|tjr|j$rt#|t%st&d t'd| j||| ( ||d|}|  t)j*t!j+t#|t,r|nt-|d || jjj|< d|j.j/vsJ |j01 |j.j/d< |j2j.j/d	 }t3|| jj4urt5d
t6||d|}	|	|j.j/d< | jj7|	 |S )Nr   r  r  Ztorch_function_fnz3torch.compile does not support strided NestedTensor)r   proxyexample_valueshould_specializesubclass_typer   rw  Ztensor_dictr  z(Wrapped Tensor must be this graph's fakeFr  r   )8r   r   r   r   r   r)  r8   Zis_fsdp_moduler}  rv  r   r/   rJ  rK  r   r   r   r   r   r   r   r   r   r   r   r   r   input_source_to_varrH  rI  r   r   r   r&   r3  r   	is_nestedr   r%   wrap_fx_proxyrg  r/  r0  ZTENSOR_MATCHr1   r   rD  rL  __dict__copyr~  r   r|  r$   r   Zadd_symbol_bindings)
r   r   r   r  Zis_duplicate_tensorZtensor_proxyr   Ztensor_variablefake_tensor_valuer  r   r   r   r     s   









	zVariableBuilder.wrap_tensorc              
   C   s>  t d usJ t|t jsJ t|  }ddlm} |jj }|r%d|j_z|	|}|r7ddl
m} ||}W n tyP } ztt| W Y d }~nd }~ww t| j||  | jjjjtdd| jt||d}d|i}	tdt| j||d	|	}
|
| jjj|< |
jjjd
 }t||d|d|d}||jjd< |
S )Nr   )_utilT)clone_preserve_stridesr  r  r   r   )
target_clsr   r~  r  r  )r   r   r   r   r  r   ) r   r   r   r1   r   Ztorch._numpyr  flagsZ	writeableZ_try_convert_to_tensorZtorch._prims_commonr  NotImplementedErrorr%   r   r   r   Zrecursive_realizer   rH  rI  rJ  rK  r   r   wrap_fx_proxy_clsrz   r  r~  rD  rL  r   )r   r   r   r  readonlyZtensor_valuer  er~  r   Znumpy_ndarray_variabler  r  r   r   r   r   Q  sT   


z"VariableBuilder.wrap_numpy_ndarrayc                 C   s  | j | jjjv r| jjj| j  S | jjj}t jr?t|t	r?|
 }t| | jjj|jj | jjjt|| jd  nt|t	rt|  st|  tstjjjrb| tj tj|dS | j  }|| jjj vrut!|d d}n| jjj | }|j"|krt#$d|||j" d |_"|| jjj |< tj%r|j"d u stj&st'j(}n| tj tj|dS |j)|| j|d}| jjj|jj | jjjt|| jd  nt*|}t|  tst+|  ,tj- d|  i}t|tj.r|/d|i | jjj0j1t23dd| j t4||  d	}t5t6f| j||d
|}	|	| jjj| j < t|  s| jj7rAt|  t8sAt9d:||  d }
t|	trM|	j;}n|	j<jj=d }t>|rl|}
|
j?| jj?u slJ d|
j? dt@|  |t|tj.|
d|d|jj=d< |	S )Nrw  )r   r   z%automatic dynamic int %s val %s != %s)r   r  r   Z	raw_valuer  r  r   )r   r~  r  zJDynamo attempts to add additional input during export: value={}, source={}r  zfake mode (z^) from fake tensor metadata doesn't match mode({self.tx.fake_mode}) from InstructionTranslatorFr  r  )Ar   r   r   Zunspec_variable_maprE  r   r  ry  r   r   Zcreate_unbacked_symintr   rM  rN  rD  rO  rP  r   rQ  r   r/   r   r2   r   Z_dynamor   rx  r   r&   r{  rJ   r   frame_stater   r   logdebugautomatic_dynamic_shapesassume_static_by_defaultr   rG  rF  tensorr'   r   r3  r   updaterH  rI  rJ  rK  r   r  r~   exportr0   AssertionErrorformatr   r~  rL  r   r|  r   )r   r   rE  Zwrapped_valuer   frame_state_entryr  r   r~  Z
unspec_varr  r  r   r   r   r;    s   

	









z,VariableBuilder.wrap_unspecialized_primitive)+r   r   r   __doc__r3   r   r   r   staticmethodr/  	lru_cacher   r   r   r   classmethodr   r   r   r  rg  r   r   r   r>   r   r   rA   r   r   r   r   r   r   r+  r,  r   r   r}  r   r   r;  __classcell__r   r   r   r   r      sD    

)   ;	
@|8r   c                 C   s~   t | tr	| j}nt | tr| j}ntd|   g }t|D ]}d }| jr1t	t
| jd|j}|t||d q t|S )Nz)Dataclass fields handling fails for type __dataclass_fields__r   )r   r   r   rQ   r  r%   r   r   r   r-   r*   r   r   ri   )objr   r2  fieldr   r   r   r   r     s   

r   c                 K   sL   | |||d|}|d u rt ddti|S t ddti|}||  |S )N)r   r~  r  r  r  r   )r  r}   r   Zinstall_global)r   r~  r  r  r   kwargsr   r   r   r   r    s   
r  c                    s  ddl m} t |sJ d|v r |d d ur  jj|d  d|jjvs/J |jjd  |} fdd}t | |d u rHt	|j dd}nSt
| ju rPnKt|tjr jrwtj  |||jj|j< W d    n1 srw   Y  d	| ttfv i}	d
|v r|d
 d usJ |d
 |	d
< t|fd i|	}t|tjrt
| jurtd| W d    n1 sw   Y  t|tjrt|tjj}
|dd}|
s|r|}nd }||}||jjd< | |}t|tjjjr|j ju r|r|ntj}|
rtjjn||d< ||d< || | |fi |S t|jjdr8|jjj dkr8t|jjj!tjj"sA|jjtj#j$krGt%|jjS |jjtjj&ksY|jjtj'j(kr]t)|S t*|tj+r}t,dd |D r}dd |D }t-|fi |S t|t.t/t0fr*||jjd< g }t1|D ]0\}}|d u r|2t3j4d&i | q|2t5|  |j6dt7j8||fi fd|i| qt|tj+rt-||fi |S t*|t.rt9|fi |S t*|t/t:frt;|fdt< i|S t*|t0rt=|fdt< i|S |j>j?dks t|ds J d|j>j? dt@| tA||j>fi |S |d u s7|jjtjBu r?t3j4d&i |S t|tjCtjDtjEfrZ||jjd< tF||fi |S tGH|jjrjtI|jjtJsw|jjdd tK D v r||jjd< tL|||jMj@fi |S tGH|jjrtI|jjtNs|jjdd tK D v r||jjd< tO||fi |S |jjdkr|jjPd kr||jjd< t3|fi |S |d urt|tNr|jjd!kr|jjPd kr||jjd< tO||fi |S t|tQr1|jjtjRtSt7j8tjTjUtjVt7jWtStjXd"tYtStjXd#tYtjZtj[f
v r1||jjd< t3j4|fi |S t\d$t]| d%|jjP d%|jj   d S )'Nr   )InstructionTranslatorBaser   r  c                    s@   t | tjrt | tst| rt|  ju s| jst| } | S r   )	r   r   r   r   Z_is_functional_tensorr   r|  r  r6   rw  r   r   r   _clone_inputb  s   z'wrap_fx_proxy_cls.<locals>._clone_inputT)Zallow_non_graph_faker   r   r   zU`example_value` needs to be a `FakeTensor`wrapped by this instance of Dynamo. Found: r  FZ
class_typespecialized_valuer   Z	set_statec                 s       | ]}t |tV  qd S r   )r   r   r  r   r   r   r    s    

z$wrap_fx_proxy_cls.<locals>.<genexpr>c                 S   s   g | ]}t |qS r   )rJ   r   r  r   r   r   r     r   z%wrap_fx_proxy_cls.<locals>.<listcomp>call_functionrm  ztorch.return_types_fieldsz	expected z. == torch.return_types or named tuple but got c                 S      g | ]\}}|j qS r   )Zcurrent_streamr   r  device_interfacer   r   r   r         c                 S   r  r   )Eventr  r   r   r   r     r  queryZcall_methodZrecord_eventZget_rankZget_world_sizeztorch.* op returned non-Tensor  r   )^Zsymbolic_convertr  r   r   r   r  rD  rL  r?   r7   r   r|  r   r   r  r>  ZDisableTorchFunctionSubclassZtracerZreal_value_cacher}   r   wrap_to_fake_tensor_and_recordr$   r   r   popZ
specializer   r   r   hasattrtargetr   r  	GeneratorrandomZset_rng_stater   Z_DisableFuncTorchcudaZ_is_in_bad_forkr   r=   r   r(  rf   r   r   setr1  r   rJ   r   r  Zcreate_proxyoperatorgetitemri   r   rb   rF   rU   r   r   r   rc   Zmanual_seedr   ZSymFloatrC  r{   r   isclassr\  r   r#   rO   r   r
   rM   opr   Zsym_intr9  _utilsZ_element_sizeseedmodru  r   Z_constrain_as_valueZ_constrain_as_sizer%   rG   )r  r   r~  r  r  r   r  Zinitial_example_valuer  r  Zis_parameterr  r  Zspecialized_propsZtensor_typesizesunpackedr  valr   r  r   r  U  sJ  	(







r  c                   @   sV   e Zd ZU eeef ed< eed< ee	e
  ed< defddZdedefdd	Zd
S )rQ  faker   constraint_dimsreturnc                 C   s   t | j| j fS r   )hashr  r   r   r   r   r   r   __hash__0  r   zTrackedFake.__hash__r   c                 C   s.   t |tr| j|ju o| j |j kS dS )NF)r   rQ  r  r   r   r   r   r   r   r   3  s   
 zTrackedFake.__eq__N)r   r   r   r   r   r   r   r3   r   DimListr   r   r  objectr   r   r   r   r   r   rQ  )  s   
 rQ  r  c              	      s@  |  }|jjj| d }|r|jnd }|r)ttjg| 	  d g| 	  ||dS t
dd |  D rHtdd |  D d g| 	  ||dS d }||jjvr]td d }t|  |_nH|jj| }|jd ur| jt|jkr~td|| j|j d |_n't|jD ]!\}}	|	d ur|  | |	krtd||| ||	 d |j|< qt| }
i   fdd	}|jjr|jjD ]%}|j|
kr||j	|j|j |jd ur|jj|
kr||jj	|j|j qg }g }t| 	 D ]}|t| d
t v }|t| dt v }|t| dt v }tjo|jd u p|j| d u }|jr)|r)td| d |j|<  |}|d u rO|r@tjs@t dd}n'|sL|rLt dd}nd }n|\}}|d urg| d| d}||jj!j"|< |#| |d usw|sw|r{tj$}n|stj%s|rtj}ntj&}|#| q||jj|< t||||dS )N)dynamic_sizesconstraint_sizesZtensor_sourcesource_to_symint_node_cachec                 s   r  r   )r   r   r   sr   r   r   r  L  s    z%_automatic_dynamic.<locals>.<genexpr>c                 S   s"   g | ]}t |trtjntjqS r   )r   r   r   rG  STATICr  r   r   r   r   N  s    z&_automatic_dynamic.<locals>.<listcomp>z!automatic dynamic %s dim %s != %sz&automatic dynamic %s size(%s) %s != %sc                    sz   |  v r5ddl m}  |  \}}||j|j@ dd}|d ur+|d u s(||ks(J |}n|}||f | < d S ||f | < d S )Nr   )StrictMinMaxConstraintF)vr	warn_only)%torch.fx.experimental.symbolic_shapesr  r  )dimconstraint_range
debug_namer  Zold_constraint_rangeZold_debug_nameZnew_constraint_rangeZnew_debug_nameZdim2constraintr   r   update_dim2constraint~  s   
z1_automatic_dynamic.<locals>.update_dim2constraintZ_dynamo_dynamic_indicesZ_dynamo_weak_dynamic_indicesZ_dynamo_static_indicesz#automatic dynamic %s marked dynamicF)r  Tz.size()[])'r   r   tracing_contexttensor_to_contextr  r  r   r   r  r  anyr   r  r   r   ndimr^  r  r  r1  r   Zexport_constraintst_idr  r  Zsharedr   r9  r  r   r  Zallow_ignore_mark_dynamicr   rE  Zsource_name_to_debug_namer   rG  r  ZDUCK)r  r   r   static_shapesr   Zprior_policyr  r  r  r  r  r  
constraintZdynamic_dimsr  Zmarked_dynamicZmarked_weak_dynamicZmarked_staticZautomatic_dynamicZconstraint_dimr  Zdim_nameZdynamicr   r  r   _automatic_dynamic;  s   	








r  r   r   c                   s2  t  tjtjjtfv st tjst rd usJ t |	 d\}}d  j
s3t |r<jjj < td t jd urMjnd d urUjnd  t fdd}|r|rk sjjt|d uryjnd  jjt   | | djj < |S  S )N)r   zwrap_to_fake %s %s %s %sc                      s   j j dS )N)r   symbolic_context)r|  Zfrom_tensorr   r  r   r  r   r   r   r     s
    z0wrap_to_fake_tensor_and_record.<locals>.<lambda>)r   stride) r   r   r   r   r   r   r   r   r@   r   r  r  r   r  r  r  r  r   r   shaper  r  rD   r)  rP  r   rQ  Ztracked_fakes_id_to_sourcer   r   r  Ztensor_weakref_to_sizes_strides)r  r   r   r   r  rW  Zfake_er   r  r   r    sP   


	r  c                   @   s*   e Zd ZdZdefddZedd ZdS )SourcelessBuildera  
    Like builder, but stateless and does not require a source. Useful for simple type->VT objects, or objects
    that are being created/evaporated during inlining (ex: consider a locally made list of tensors we then iterate over
    .), such a list should not show up as an artifact from inputs, nor in reconstruction, nor in the graph. However,
    there may be reasons to represent it as a ListVariable internally.

    NOTE - Objects produced here are born UNGUARDED due to the nature of sources!

    NOTE - This class is very new! It will have some rough edges, but it was created to stem the bleeding of giant
    if/else type->VariableTracker trees that were cropping up all over dynamo.
    r  c                    sb  t |tr|S t |tjrt|S t|rt|S t	|r#t
|S t|r4t|r0d jj_t|S t |tjr>t|S t |tjrHt|S t |ttjfrTt|S t |trkt fdd| D tt dS t |trt  fdd|D t dS t |t!t"frt#$t|}| fdd|D t dS t |tj%rt&|S t'dt|  d S )	NTc                    s   i | ]
\}}| |qS r   r   )r   r   r  r   r   r   r   9  s    z.SourcelessBuilder.__call__.<locals>.<dictcomp>rl  c                       g | ]} |qS r   r   r  r   r   r   r   ?  r   z.SourcelessBuilder.__call__.<locals>.<listcomp>c                    r  r   r   r  r   r   r   r   C  r   z&Unexpected type in sourceless builder )(r   rH   r   _HAS_DEFAULT_FACTORY_CLASSr   rJ   r   r  wrap_constant_literalr    rI   r   r"   r   r   rT  r   r7  rU  r]   r  r  rK   r   abcABCMetar   r%  rP   r2  rF   r  rU   r   r   ra   Zcls_forr[  rq   r%   )r   r   r   r   r   r   r   r   #  sD   





zSourcelessBuilder.__call__c                 C   s   t | sJ t j| dS )Nrw  )rJ   r   r   rw  r   r   r   r  H  s   z'SourcelessBuilder.wrap_constant_literalN)r   r   r   r  rH   r   r  r  r   r   r   r   r    s
    %r  )NN)r  r   r?  r   r  r/  r   loggingr  rJ  r#  r7  typingr   r   r   r   numpyr   ModuleNotFoundErrorr   r   Ztorch._guardsr   r   Z
torch._opsr	   Ztorch._streambaser
   r   Ztorch._subclasses.fake_tensorr   r   r   r  r   r   r   r   r   r   Ztorch.fx.immutable_collectionsr   Z$torch.nested._internal.nested_tensorr   Ztorch.utils._python_dispatchr   Ztorch.utils.weakr   r  r   r   r   r   r   Zallowed_functionsr   r    r!   r"   r  r#   excr$   r%   r   r&   r'   r(   r   r)   r   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r!  r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rc  rF   rG   rH   builtinrI   ZconstantrJ   rK   Zctx_managerrL   rM   rN   rO   ZdictsrP   rQ   rR   rS   rT   rU   ru  rV   rW   rX   rY   Z	functionsrZ   r[   r\   r]   r^   Zhigher_order_opsr_   Zlazyr`   listsra   rb   rc   rd   re   rf   rg   rh   ri   miscrj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   Z	nn_modulerw   rx   Z	optimizerry   r  rz   r{   r|   r}   r~   r   r   Ztorch_functionr   r   Zuser_definedr   r   r   	getLoggerr   r  r  r   	dataclassr   r   r   r   r  r  rQ  r  r   r  r  r   r   r   r   <module>   s     4H ,<
,        2
=
 U &6