o I&is, @sUddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z mZmZddlmZdZe e jed<zddlZWn eyUdZYnwddlZddlmZddlmZddlmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'dd l(m)Z)ej*Z*d Z+e,e-Z.d dZ/ddZ0ddZ1ddZ2de3fddZ4ddZ5ddZ6ddZ7de3fddZ8dSd!e j9ddfd"d#Z:de e'fd$d%Z;Gd&d'd'Zde3fd,d-Z?de3fd.d/Z@de3fd0d1ZAdTd2d3ZBd4ejCfd5d6ZDdUd9d:ZEejFd;dfdejHd?ee3ejIfd@eGf dAdBZJdCdDZKddEdFdGZLdHdIZMdJdKZNdLdMZOdNdOZPdVdQdRZQdS)WN)ListOptionalSequenceUnion)patchnp)fx) OutputGraph)config eval_frameoptimize_assertreset)create_instruction debug_checks is_generatortransform_code_object)CheckFunctionManager GuardedCode)samecCs |durdS||jSN)detachcloneZrequires_grad_ requires_grad)xr@C:\wamp64\www\opt\env\Lib\site-packages\torch/_dynamo/testing.pyclone_me(srcstfdd}|S)Ncs"dtjvr td|i|S)NZPYTEST_CURRENT_TESTzdoes not work under pytest)osenvironunittestZSkipTest)argskwargsfnrrwrapped/s  zskip_if_pytest..wrapped functoolswraps)r%r&rr$rskip_if_pytest.sr*cCt|tjsJ|jjSr) isinstancer OptimizedModule _orig_modnamed_parametersmodrrr%named_parameters_for_optimized_module8r2cCr+r)r,r r-r. named_buffersr0rrr"named_buffers_for_optimized_module=r3r5returncCstdd|S)Nz ^_orig_mod[.])resub)namerrrremove_optimized_module_prefixBsr;cCs&g}||||t}t}|D]'\}}t|tjr$t|}|} |j} |jdur3t |} | ||d<| ||<q||||t} | D]\}} t|tjr]t|}| | |<qO|| |D]'} t| t t fr| D]}t|tj r||jqtqit| tj r|| jqi|S)Nz.grad)appenddictr/r,r r-r;gradtorchZ zeros_liker4tuplelistTensor)modelZ predictionZlossexample_inputsresultsZgradsparamsr:paramZ param_copyr>buffersbufferZexampleinprrrcollect_resultsFsB                rKcCsZt|tjr |jSt|ttfrtdd|DS|durdSt|tr&dStdt |)Ncss|]}t|VqdSr)requires_bwd_pass.0rrrr rsz$requires_bwd_pass..FDon't know how to reduce) r,r?rBrrAr@anyintNotImplementedErrortypeoutrrrrLns  rLcCst|tjr||St|ttfr"tdd|Dt|St|j dvr.t |j St|j dkr:|j St|t rPtdd|Dt|Stdt|)z/Reduce the output of a model to get scalar losscSg|]}t|qSrreduce_to_scalar_lossrMrrr z)reduce_to_scalar_loss..)ZMaskedLMOutputZSeq2SeqLMOutputZ!CausalLMOutputWithCrossAttentionsZSquashedNormalcSrWrrX)rNvaluerrrrZr[rP)r,r?rBsumZnumelrAr@lenrT__name__rYZlogitsmeanr=valueskeysrSrUrrrrYzs    rYcCs0tjtjtd}tj|st||S)Nz../debug)rpathjoindirname__file__existsmkdir)rcrrr debug_dirs  rir7codec Cslttjt|d!}|t|dt|d|dWddS1s/wYdS)Nwz  ) openrrcrdriwritedisBytecodeinfo)r:rjextrafdrrr debug_dumps ("rtc Cs`dd}t|jr dSt|jt|j|}tidddddditt|jd }t|t|j S)zused to debug jump updatescSs$|dtd|dtddS)NrNOP)insertr)Z instructions code_optionsrrr insert_nopssz&debug_insert_nops..insert_nopsNFZ_idr) rwZ compiler_fnZroot_txZexportZexport_constraintsZ frame_stateZ local_scopeZ global_scopef_code) rryrrr localsglobalsrrZcheck_fn)frameZ cache_sizehooks_rxrjgraphrrrdebug_insert_nopss"    rc@s8eZdZddZdejjdeejfddZ ddZ d S) CompileCountercCd|_d|_dSNr frame_countop_countselfrrr__init__ zCompileCounter.__init__gmrDcCs:|jd7_|jjD]}d|jvr|jd7_q |jS)Nr call)rrnodesoprforward)rrrDnoderrr__call__s   zCompileCounter.__call__cCrrrrrrrclearrzCompileCounter.clearN) r_ __module__ __qualname__rr?r GraphModulerrBrrrrrrrs rc@0eZdZddZdejjdeejfddZ dS)CompileCounterWithBackendcCsd|_d|_||_g|_dSr)rrbackendgraphs)rrrrrrs z"CompileCounterWithBackend.__init__rrDcCs\ddlm}|jd7_|jjD]}d|jvr|jd7_q|j|||j ||S)Nr )lookup_backendr) Zbackends.registryrrrrrrrr<r)rrrDrrrrrrs    z"CompileCounterWithBackend.__call__N r_rrrr?rrrrBrrrrrrs rc@r)EagerAndRecordGraphscCs g|_dSr)rrrrrrs zEagerAndRecordGraphs.__init__rrDcCs|j||Sr)rr<)rrrDrrrrs zEagerAndRecordGraphs.__call__Nrrrrrrs rcCst|}tdd|S)Nz (?m)^ *#.*\n?r7)strr8r9rjrrr strip_commentsrcCsddd|dDS)NrlcSsg|]}|qSr)rstrip)rNlinerrrrZr[z)remove_trailing_space..)rdsplitrrrrremove_trailing_spacesrcCs tt|Sr)rr)Zgm_strrrr normalize_gms rcCstjs |dur |}t}ddt|D}ddt|D}||}||} tt||} | |} | |} | |} | |}t|t| ||t| ||t| | |t|| ||j d|durv||j |dSdS)NcSg|]}tddqS r?randnrNr~rrrrZz!standard_test..cSrrrrrrrrZrr ) r Zassume_static_by_defaultrrangerr assertTruer assertEqualrr)rr%nargsZ expected_opsZexpected_ops_dynamicactualZargs1Zargs2Zcorrect1Zcorrect2Zopt_fnZval1aZval2aZval1bZval2brrr standard_tests, rrcCs|jSr)r)rrDrrrdummy_fx_compilesrT皙?cCs.|sdS||kr|ddS|dd|dS)NERRORz.3fzx SAMEzx p=z.2fr)ZspeedupZpvalueZ is_correctZpvalue_thresholdrrrformat_speedups  rcpusizestridedtypedevice extra_sizecCsXtddt||Dd|}|jrtj|||d}n tj|g||d}t|||S)Ncss |] \}}|d|VqdS)r Nr)rNshaperrrrrO%szrand_strided..r )rr)rrr)r]zipZis_floating_pointr?rZzerosZ as_strided)rrrrrZ needed_sizerIrrr rand_stridedsrcstfdd}|S)Nc s^t!}D]\}}}|t|||q|i|WdS1s(wYdSr) contextlib ExitStack enter_contextrobject)r"r#stackmoduleattrvalr%patchesrr_fn1s  $z"_make_fn_with_patches.._fnr')r%rrrrr_make_fn_with_patches0sr) xfail_propc Gst||j|ji}|j|_t|D]N}|drTt||}t|s.t||t||q||}t |g|R} || _|durMt ||rMt | } t||| qt ||sbt||t||q|S)NZtest_) rTr_ __bases__rdir startswithgetattrcallablesetattrrhasattrr!ZexpectedFailure) clsZ cls_prefixZ fn_suffixrrZDummyTestClassr:r%new_nameZnew_fnrrrmake_test_cls_with_patches<s$      rcCstjdkr|St|S)N)r )sys version_infor!skipr$rrrskipIfNotPy311Ts  rcC d|_|SNT)Z_expected_failure_dynamicr$rrrexpectedFailureDynamic\rcCrr)Z!_expected_failure_codegen_dynamicr$rrrexpectedFailureCodegenDynamicbrrcCrr)Z!_expected_failure_dynamic_wrapperr$rrrexpectedFailureDynamicWrapperhrrFcCsVtdtdtrtjd|r)ddlmm}|dt | dSdS)Ni9r) r?Z manual_seedrandomseedrZtorch_xla.core.xla_modelcoreZ xla_modelZ set_rng_staterZ xla_device)Zuse_xlaZxmrrrreset_rng_statems   r)r7)NN)Tr)F)Rrror(loggingZos.pathrrr8rtypesr!typingrrrrZ unittest.mockrr ModuleType__annotations__numpyModuleNotFoundErrorr?rZtorch._dynamo.output_graphr r7r r r rZbytecode_transformationrrrrZguardsrrutilsr unsupportedthree getLoggerr_logrr*r2r5rr;rKrLrYriCodeTypertrrrrrrrrrrrZfloat32rRrrrrrrrrrrrrrrs         (