o <╞&id}у @s■UddlZddlZddlZddlmZddlmZddlTddlm Z gZ eee d<iZeeefe d<d d ДZddДZd dДZdeddfddДZdededededdf ddДZddДZddДZddДZeаeeбZdeefdd ДZd!d"ДZd#d$ДZd%d&ДZ d'd(ДZ!eej"j#j$j%ej"j#j&j%ej"j#j'j%ej"j#j(j%ej"j#j)j%ej"j#j*j%gd)Гd*d+ДГZ+eej"j,j-j%d)Гd,d-ДГZ.eej"j#j/j%gd)Гd.d/ДГZ0eej"j#j1j%d)Гd0d1ДГZ2eej"j#j1j3d2Гe2Гeej"j#j4j%d3Гd4d5ДГZ5eej"j#j6j%d6Гd7d8ДГZ7eej"j#j8j%d9Гd:d;ДГZ9eej"j#j:j%ej"j#j;j%ej"j#jj%d<ГeГeej"j#j?j%d=Гd>d?ДГZ@eej"j#jAj%d@ГdAdBДГZBeej"j#jCjDdCГdDdEДГZEeej"j#jFjGdFГdGdHДГZHeej"j#jIj%dIГdJdKДГZJeej"j#jKjLdLГdMdNДГZMeej"j#jNj%dOГdPdQДГZOeej"j#jPj%dRГdSdTДГZQeej"j#jRj%dUГdVdWДГZSeej"j#jTj%dXГdYdZДГZUeej"j#jVj%d[Гd\d]ДГZWeej"j#jXjYd^Гd_d`ДГZZeej"j#j[j%daГdbdcДГZ\eej"j#j]j%daГdddeДГZ^eej"j#j_j%dfГdgdhДГZ`eej"j#jajbdiГdjdkДГZceej"j#jdjLdlГdmdnДГZeeej"j#jfj%doГdpdqДГZgeej"j#jhj%drГdsdtДГZieej"j#jjj%duГdvdwДГZkeej"j#jljLdxГdydzДГZmeej"j#jnjGd{Гd|d}ДГZoeej"j#jpj%d~ГddАДГZqeej"j#jrj'dБГdВdГДГZseej"j#jtj%dRГdДdЕДГZueej"j#jvj%dЖГdЗdИДГZwdS)ЙщN)┌#jagged_scaled_dot_product_attentionщ)┌NestedTensor)┌*)┌normalize_function┌__all__┌JAGGED_OPS_TABLEcCs(|dkr||ks JВ|dkrdS|dS)Nrщrй)┌ndim┌dimr r ·EC:\wamp64\www\opt\env\Lib\site-packages\torch/nested/_internal/ops.py┌_outer_to_inner_dimsrcCs6ddlm}|||Г}|dkrt|ЫdЭГВt||ГS)Nrй┌canonicalize_dimsr z4(): not supported for NestedTensor on dim=0 or dim=1)┌torch._prims_commonr┌RuntimeErrorr)rr┌op_namer┌wrappedr r r ┌_wrap_jagged_dims rcs|ddlmЙЗЗfddД|DГ}d|v}d|v}||Ar1|r dnd\}}t|Ыd|Ыd |Ыd ЭГВtЗfddД|DГГ|fS) Nrrcsg|]}ИИ|ГСqSr r й┌.0┌dйrrr r ┌ 'єz%_wrap_jagged_dims..r)┌batch┌ragged)rrz(): applying over the z dimension, but not the z, dimension is not supported for NestedTensorc3s"Б|]}|dkrtИ|ГVqdS)rN)rr)rr r ┌ 3sА z$_wrap_jagged_dims..)rrr┌tuple)r┌dimsrZwrapped_dimsZzero_in_dimsZone_in_dims┌applyZ not_applyr rr ┌_wrap_jagged_dims"s ■r"┌ schema_str┌returncOs^|аdб}tddД|DГГ}t|Г|}t|Г|kr"t|Гt|Гks;td|jЫd|Ыd|Ыdt|ГЫdt|ГЫd ЭГВd dДddДd dДddДdЬ}t|ГD]^\}} | аdб\} }|аdб}|rf|ddЕn|} | |абvrutd| ЫЭГВ|t|ГkrМ|sЛtd|jЫd|Ыd| ЫЭГВqN|| ||Гsмtd|jЫd|Ыd| Ыd|Ыdt ||ГЫЭ ГВqNdS)Nz, cSsg|]}|аdбСqS)·?)┌endswithйr┌xr r r r:rz check_schema..· NestedTensor ·(z): expected at least z arguments and at most z arguments, but got: z argumentscSst|tjГot|tГSйN)┌ isinstance┌torch┌Tensorrйr(r r r ┌Eєzcheck_schema..cSst|tГo |jduSr+)r,r┌_lengthsr/r r r r0Fs cSs t|tГSr+)r,rr/r r r r0Hs cSsdS)NTr r/r r r r0Ks)┌tZjtZjt_all┌anyz: r%щ zUnknown arg type: z) missing required argument: z): z should be of type z, but got: ) ┌split┌sum┌len┌ ValueError┌__name__┌ enumerater&┌keys┌AssertionError┌type)r#┌func┌args┌kwargsZnamed_arg_typesZnum_optional_argsZmin_argsZarg_type_check_fns┌iZnamed_arg_type┌nameZarg_typeZis_optionalZnormalized_arg_typer r r ┌check_schema8sP ■ ∙ ёrD┌a┌a_name┌b┌b_namecCs:|j|j|j|jkrtd|jЫd|Ыd|ЫdЭГВdS)Nr)z: expected · and z' to have the same exact offsets tensor.)┌_size┌_ragged_idxrr:)r?rErFrGrHr r r ┌check_ragged_dim_samecs rLcCs,|jd}t|jd|ЕГt|d|ЕГkS)Nr)rK┌listrJ)┌nt┌size┌endr r r ┌raggedness_matchesps "rQcCs*|jddkr|аdб}|jddks|SйNrr)┌shapeZsqueeze)r3r r r ┌squeeze_leading_onesus rTcs4tИtГsИgЙtИtГsИgЙЗЗЗfddД}|S)Ncs2ИD]}ЗЗfddД}ИD]}||Г||<q qИS)NcsЗЗЗfddД}|S)Ncs.tИИg|вRi|дОИИg|вRi|дОSr+)rD)r@rA)┌aten_opr?r#r r ┌innerЧsz@register_func..wrapper..get_inner..innerr )rUrV)r?r#)rUr ┌ get_innerЦsz1register_func..wrapper..get_innerr )r?rUrW┌tableй┌aten_opsr#┌tables)r?r ┌wrapperУs zregister_func..wrapper)r,rM)r[rZr#r\r rYr ┌ register_funcНs r]cOs|tа|dб}|dur|Stjj|jvr.rr zlhs: any, rhs: any)r┌getr-┌TagZ pointwise┌tagsr7┌ functools┌partial┌jagged_unary_pointwiserD┌jagged_binary_pointwise)r?r@rAZ dispatch_funcZnum_tensor_argsr r r ┌ lookup_jaggedзsrecCs|аб|j|jdЬ}|S)Nй┌offsetsZ_max_seqlenZ_min_seqlenrf)┌argrAr r r ┌extract_kwargs╣s ¤ricOs8t||djg|ddЕвRi|дОfit|dГдОSrR)r┌_valuesriйr?r@rAr r r rc┬s " rccOsь|d|d}}t|tГst|tГsJВd|jЫd|jЫd|jЫЭ}t|tГrQt|tГrQt||jГrMt||j|jg|ddЕвRi|дОfit|ГдОSt|ГВt|tГ}|r\t|Гnt|Г}|rf||fn||f\}} | аб|абkrxt dГВt | Г} |аб| абdkrк|rН|j| fn| |jf\}}t|||g|ddЕвRi|дОfi|дОS|аб|абkrЄ|jd|jdkr└t|ГВg} t|аб|абГD]\}}| а |||g|ddЕвRi|дОбq╦tj| ddН}t|fi|дОSt|ГВ) Nrrz&cannot call binary pointwise function z with inputs of shapes rIr z+NYI: broadcasting NT with T with larger dim)r)r,rr:rSrQrjrirr┌NotImplementedErrorrT┌zip┌unbind┌appendr-┌cat)r?r@rArErGZmismatch_error_msgZa_is_ntZextracted_kwargsrNr3Z t_squeezed┌lhs┌rhsZoutputsZa_compZb_comp┌ new_valuesr r r rd╚sD " ,(rdcOsи|tjjjurt|i|дОS|jdkrPdddД}t|||ddН\}}|аdб}t|а б|d dГ|d <t|а б|d dГ|d <t ||jfi|дОfit|ГдОSt |ГВ)N┌flattenrr5cSsdSr+r )┌input┌ start_dim┌end_dimr r r ┌_flatten_sigsz+jagged_torch_function.._flatten_sigTйr@rAZnormalize_to_only_use_kwargsrurvrw)rr5)r-Z_CZ_nnZscaled_dot_product_attentionrr:r┌poprrrrjrirl)r?r@rArx┌_┌ new_kwargs┌inpr r r ┌jagged_torch_functions "r~zself: jt_allcOs▐|tjjjjkr dS|tjjjjkr|djS|tjjjjkr&t|djГS|tjjj jkrQ|dj durJtt|dj Гt а|djddЕбГS|djабS|tjjjjkr^|djS|tjjjjkrm|djабSdS)NFrr )r-┌ops┌aten┌is_non_overlapping_and_dense┌default┌sym_sizerJrr8┌ sym_numelr2┌intr7┌math┌prodrjZnumel┌ sym_strideZ_strides┌sym_storage_offsetZstorage_offsetrkr r r ┌tensor_attr_supported_getter"s * rКcOstjSr+)r-Zjaggedrkr r r ┌prim_layout_defaultCsrЛcOs|tjjjjkrtdГВdS)NznNestedTensors does not support directly calling torch.ops.aten.size please use `nested_tensor.size()` instead.)r-rrАrOrВrrkr r r ┌tensor_attr_unsupported_getterHs rМcOsАddlm}t|||ddН\}}|аdб}|абdurdS|jdkr$dS|аdtjб|d<|dtj kr6dS||а бfi|дОS) Nr)┌is_contiguous_for_memory_formatTryruFr┌ memory_format)rrНrrz┌lengthsrKr^r-Zcontiguous_formatZpreserve_format┌values)r?r@rArНr{r|r}r r r ┌is_contiguous_generalTs rСz!self: jt_all, memory_format: any?zinput: jt, weight: t, bias: t?cOsPt|||ddН\}}|аdб}|d}|d}t||jfi|дОfit|ГдОS)NTryru┌weight┌biasйrrzrrjri)r?r@rAr{r|r}rТrУr r r ┌linear_defaultrs "rХz6self: jt, grad_output: jt, weight: t, output_mask: anycOsАt|||ddН\}}|аdб}|аdб}|аdб}t||d|dГttа|j|бfit|ГдО}tа|jj|jб} d} || | fS)NTryru┌grad_outputrТ┌self) rrzrLrr-┌mmrjri┌T)r?r@rAr{r|r}rЦrТZdsZdw┌dbr r r ┌linear_backward_defaults rЫzself: jtcOsNt|||ddН\}}|аdб}|аdб||jfi|дО}t|fit|ГдОS)NTryru┌layoutйrrzrjrri)r?r@rAr{r|r}rsr r r ┌to_copy_defaultХs rЮz&self: jt, dim: any, half_to_float: anyz!self: jt, float: any, train: any?cOs\t|||ddН\}}|аdб}||jfi|дО\}}t|fit|ГдОt|fit|ГдОfSйNTryrurЭ)r?r@rAr{r|r}Zout1Zout2r r r ┌native_dropout_default┤s ■rаz%grad_output: jt, mask: jt, scale: anycOsNt|||ddН\}}|аdб}|аdб}t||j|jfi|дОfit|ГдОS)NTryrЦ┌maskrФ)r?r@rAr{r|rЦrбr r r ┌native_dropout_backward_default─s ■rвz!self: jt, dim: any, keepdim: any?cOsrt|||ddН\}}|аdб}|dstdГВ|d}tt|jГ|dГ|d<t||jfi|дОfit|dГдОS) NTryru┌keepdimz1prod(): keepdim=True must be set for NestedTensorrrЗr) rrzrrr8rSrrjri)r?r@rAr{r|r}rr r r ┌prod_dim_int╘s &rдz#self: jt, split_size: any, dim: anycsZt|||ddН\}}|аdбЙtИаб|ddГ|d<tЗfddД|Иjfi|дОDГГS)NTryrurr6c3s&Б|]}tdd|itИГдОVqdS)rРNr йrrir'йr}r r rёs А zsplit_tensor..)rrzrrrrjйr?r@rAr{r|r rжr ┌split_tensorхs ■rиz$self: jt, split_sizes: any, dim: anycsVt|||ddН\}}|аdбЙtИаб|ddГ|d<ЗfddД|Иjfi|дОDГS)NTryrur┌split_with_sizescs"g|] }tdd|itИГдОСqS)rРr rеr'rжr r rs z,split_with_sizes_default..)rrzrrrjrзr rжr ┌split_with_sizes_defaultўs ■rкzself: jt_all, dim: any?csЬt|||ddН\}}|d}|dkrtdГВ|аdб}|абЙ|абЙ|абЙ|jdkr0tdГВИdur>tаИИа ба ббSЗЗЗfd d ДtИjdГDГS)NTryrrz2unbind(): only supported for NestedTensor on dim=0rurzDunbind(): only supported for NestedTensor when jagged dimension is 1cs(g|]}ИИ|И|И|ЕСqSr r )rrBйrПrgrРr r r"s zunbind_int..) rrrzrРrgrПrKr-r6┌diff┌tolist┌rangerS)r?r@rAr{r|rr}r rлr ┌ unbind_ints& rпzself: jt, dim: anyc Oslt|||ddН\}}|аdб}|j}|j}|d}tt|jГd|dГ|d<t||fi|дОfit|ГдОS)NTryrurr┌ unsqueeze) rrzrjrgrr8rSrri) r?r@rAr{r|r}rРrgrr r r ┌unsqueeze_default's r▒ztensors: any, dim: anycsвt|||ddН\}}|аdб}ddД|DГ}t|ГdksJВ|dЙЗfddД|DГ}|d}ttИjГ|d Г|d<t|d dД|DГfi|дОfit|dГдОS)NTry┌tensorscSsg|]}|jr|СqSr )┌ is_nestedйrr3r r r r@rzcat_default..rcs g|]}|jr |n|аИбСqSr )r│┌ expand_asr┤й┌firstr r rCs rrpcSєg|]}|jСqSr йrjr┤r r r rJє)rrzr8rrSrri)r?r@rAr{r|r▓┌nestedrr r╢r ┌cat_default7s r╝zself: jt, other: anycOs╛t|||ddН\}}|аdб}|аdб}|jr,|js,t||j|fi|дОfit|ГдОS|jrS|jrS|абdkrS|абdkrSt||jГrSt||j|jГfit|ГдОSt d|jЫd|jЫЭГВ)NTryru┌otherщz1matmul(): not supported between inputs of shapes rI) rrzr│rrjrirrQrSrйr?r@rAr{r|r}r╜r r r ┌matmul_defaultNs" $ r└z#self: jt, size: any, implicit: any?cOsКt|||ddН\}}|аdб}|d}d|vs|аdбrJВt||Гs.td|jЫd|ЫЭГВdg|d dЕв}t||j|Гfit|ГдОS) NTryrurOZimplicitzexpand(): cannot expand shape z -> r5r )rrzrQrrSrrjri)r?r@rAr{r|r}rOZ expand_argr r r ┌expand_defaultes r┴zself: t, other: jtcOsDt|||ddН\}}|аdб}|аdб}t|||jГfit|ГдОS)NTryrur╜rФr┐r r r ┌expand_as_defaultxs r┬z"condition: jt, self: jt, other: jtcOs~t|||ddН\}}|аdб}|аdб}|аdб}|j|jkr'|jks*JВJВt||j|j|jfi|дОfit|ГдОS)NTry┌ conditionrur╜)rrzrSrrjri)r?r@rAr{r|r├r}r╜r r r ┌ where_selfДs " ■r─zself: jt, device: any?cOє@t|||ddН\}}|аdб}t||jfi|дОfit|ГдОSrЯrФйr?r@rAr{r|r}r r r ┌_pin_memory_defaultЦs "r╟cOs0t|||ddН\}}|аdб}||jfi|дОSrЯ)rrzrjr╞r r r ┌is_pinned_defaultбs r╚zself: jt, other: jtcOs|dj|djkSrR)rJrkr r r ┌is_same_size_defaultмsr╔z/self: jt, dim: any?, keepdim: any?, dtype: any?cOsЦt|||ddН\}}|аdб}|jdksJВt|аб|ddГ\|d<}|s7t||jfi|дОfit|ГдОS||jfi|дО}|drI|аdб}|S) NTryrurrr7rгr) rrzrKr"rrrjrir░)r?r@rAr{r|r}Zragged_reduced_away┌outr r r ┌sum_dim_IntList▒s " r╦zself: jt, dim0: any, dim1: anyc Ost|||ddН\}}ddlm}|аdб}||аб|d|dfГ\}}||jks.||jkrd|dks6|dkr:tdГВ||jkrB|} n|} t|аба t t|jГ|Гt t|jГ|Гбfit |Гдd | iдОSt|аб|dd Г|d<t|аб|dd Г|d<t||jfi|дОfit |ГдОS)NTryrrru┌dim0┌dim1z?Transpose is not supported on the batch dimension for jagged NTrK┌ transpose)rrrrzrrKr9rrРr╬rr8rJrirrj) r?r@rAr{r|rr}r╠r═Zto_dimr r r ┌ transpose_int╩s6 ■ √· "r╧zself: jt, size: anycOsКt|||ddН\}}|аdб}|аdб}t|Гdkst||Гs*td|jЫd|ЫЭГВ|jjdg|d dЕ}t||j|Гfit|ГдОS) NTryrurOr╛zview(): cannot view shape z as rr ) rrzr8rQrrSrjrri)r?r@rAr{r|r}rOZjagged_sizer r r ┌view_defaultЁs r╨zDinput: jt, normalized_shape: any, weight: any?, bias: any?, eps: anyc OsАt|||ddН\}}|аdб}|d}|абdks#|абt|Гdkr'tdГВ||jfi|дО\}}} t|fit|ГдО|| fS)NTryru┌normalized_shaper╛r zJlayer_norm(): normalizing over ragged dim not supported for nested tensors)rrzrr8rrjrri) r?r@rAr{r|r}r╤┌output┌meanZstdr r r ┌native_layer_norm_defaults r╘zpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc Ospt|||ddН\}}|аdб}|аdб}||j|jfi|дО\}}} |dur+d|| fSt|fit|ГдО|| fS)NTry┌grad_outrurЭ) r?r@rAr{r|r╒r}Zd_inputZd_gammaZd_betar r r ┌"native_layer_norm_backward_defaults r╓zself: jt, dim: any, index: anycOєXt|||ddН\}}|аdб}t|аб|ddГ|d<t||jfi|дОfit|ГдОS)NTryrur┌selectйrrzrrrrjrir╞r r r ┌ select_int(s "r┌z7self: jt, dim: any?, start: any?, end: any?, step: any?cOr╫)NTryrur┌slicer┘r╞r r r ┌slice_tensor4s "r▄z{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anycOr┼rЯrФr╞r r r ┌convolution_defaultCs "r▌z.self: jt, dim: any?, keepdim: any, dtype: any?cOs^t|||ddН\}}|аdб}t|аб|dddГg|d<t||jfi|дОfit|ГдОS)NTryrurrr╙r┘r╞r r r ┌mean_dimRs "r▐cOs╝t|||ddН\}}|аdб}|D]%}t|tГstdГВ|аб|dабkr*tdГВt||djГs6tdГВqt|dабd|d d Г|d <t|ddД|DГfi|дОfit |dГдОS) NTryr▓z+stack(): expected all nested tensors inputsrz9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurerr┌stackcSr╕r r╣r┤r r r r|r║z!stack_default..) rrzr,rrrrQrSrri)r?r@rAr{r|r▓r3r r r ┌ stack_defaultas0 rрzQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?cOsLt|||ddН\}}|аdб}|аdб}t|||jfi|дОfit|ГдОS)NTry┌indicesrТrФ)r?r@rAr{r|rсrТr r r ┌embedding_defaultАs rт)xrarЖr-Ztorch.nested._internal.sdparZ nested_tensorr┌typingZtorch.fx.operator_schemasrr┌List┌Any┌__annotations__r┌Dictrrr"┌strrDrLrQrTr]rbZregister_jagged_func┌Optional┌Callablererircrdr~rrАrБrВrГrrДrИrЙrКZprimrЬrЛrOrМZ is_contiguousrСrОZlinearrХZlinear_backwardrЫZ_to_copyrЮZ ones_likeZ zeros_likeZ randn_like┌detachZ_softmaxZnative_dropoutrаZnative_dropout_backwardrвrЗZdim_intrдr6r.rиrйrкrnrЕrпr░r▒rpr╝┌matmulr└┌expandr┴r╡r┬┌whererЧr─Z_pin_memoryr╟┌ is_pinnedr╚Zis_same_sizer╔r7Zdim_IntListr╦r╬r╧┌viewr╨Znative_layer_normr╘Znative_layer_norm_backwardr╓r╪r┌r█r▄Zconvolutionr▌r╙r▐r▀rрZ embeddingrтr r r r ┌sH + ■ = ·ў ■ ■ ■ №∙° ■ ■ % ■ ■ ■ ■ ■