o <&ib5!@sddlZddlmZddlmZmZmZmZmZmZm Z m Z m Z ddl m Z ddlmZmZddgZd%d d ZGd ddeZd ed e d ed e d e_     d&deedeedeedeedeedeedeedededededededededef d dZdeedeedeedeedeedeedededededededededefd!d"Zdeedeedeedeedeedeedededededededededefd#d$ZdS)'N)Tensor) Optimizer_use_grad_for_differentiable _get_value_default_to_fused_or_foreach_differentiable_doc _foreach_doc _maximize_doc_capturable_doc _view_as_real) is_compiling)ListOptionalASGDasgdcCst|tjs tj||dS|S)Ndevice) isinstancetorchrtensor)xrr;C:\wamp64\www\opt\env\Lib\site-packages\torch/optim/asgd.py _to_tensor s rc sheZdZ         ddeed ed ed effd d ZfddZddZedddZ Z S)r{Gz?-C6??.ArNFforeachmaximizedifferentiable capturablec sld|ks td|d|kstd||dur | r tdt|||||||| | d } t|| dS)NgzInvalid learning rate: zInvalid weight_decay value: F0Capturable not supported with single tensor ASGD) lrlambdalphat0 weight_decayrr r!r") ValueErrordictsuper__init__) selfparamsr$r%r&r'r(rr r!r"defaults __class__rrr,s$   z ASGD.__init__cs2t||jD]}|dd|dd|dd|ddq t|j}t|dko9t |dd}|sO|D]}tj t |dtj d|d<q>t|dko]t |dd }|sq|D]}tj |d tj d|d <qbt|dkot |dd }|s|D]}tj t |d tj d|d <qdSdS) Nrr Fr!r"rstep)dtypeetamu) r+ __setstate__ param_groups setdefaultliststatevalueslenrZ is_tensorrfloatfloat32)r-r:groupZ state_valuesZstep_is_tensorsZ eta_is_tensorZ mu_is_tensorr0rrr64s6        zASGD.__setstate__c Csd}|dD]x} | jdur~|t| O}|| | jjr!td|| j|j| } t| dkrbtjd| j tj d| d<tj |d| j tj d| d <tj d| j tj d| d <tj | tjd | d <|| d || d || d || dq|S) NFr.z&ASGD does not support sparse gradientsrr)rr3r2r$r4r5)Z memory_formatax)gradr is_complexappendZ is_sparse RuntimeErrorr:r<Zzerosrr>rZonesZ zeros_likeZpreserve_format) r-r?params_with_gradgradsmusaxsetas state_steps has_complexpr:rrr _init_groupOs,       zASGD._init_groupc Csd}|durt |}Wdn1swY|jD]?}g}g}g}g}g}g} |||||||| } t|||||| |d|d|d|d|d|d|d|d |d | d q |S) zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr%r$r'r&r(rr r!r") r%r$r'r&r(rr r!r"rL)rZ enable_gradr7rNr) r-closureZlossr?rFrGrHrIrJrKrLrrrr2is@  z ASGD.step) rrrrrNFFFN) __name__ __module__ __qualname__rboolr,r6rNrr2 __classcell__rrr0rrs.    "ahImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z z For ASGD, capturable is only supported when foreach is True. .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 Fr.rGrIrHrJrKrr r!r"rLr%r$r'r&r(c Cs|dur t||dd\}}|rtjrtd|r"tjs"t}n | r+ts+tdt}|||||||| | | ||||| | ddS)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF)Z use_fusedz6torch.jit.script not supported with foreach optimizersr#) r%r$r'r&r(r r!r"rL)rrZjitZ is_scriptingrE_multi_tensor_asgdr _single_tensor_asgd)r.rGrIrHrJrKrr r!r"rLr%r$r'r&r(_funcrrrrs4  c Cs>t|D]\}}||}| s|n| }||}||}||}||}t|r7t|}t|}t|}|d7}t|}| dkrJ|j|| d}t|}|d|||j|| dtsh| dkrt|| | |n| |t |d|||| }| |t dtd||}| |qdS)Nrrr&) enumeraterrCZ view_as_realraddZmul_Zadd_r itemsubmulZcopy_rmax)r.rGrIrHrJrKr%r$r'r&r(r r!r"rLiparamrBr5rAr4Zstep_tr2Z eta_valuenew_etanew_murrrrWs4       rWc CsNt|dkrdS| rJdt||||||g}|D]\\}}\\}}}}}}}| r3t|}t|}|r?t||||djrRtj |tj dddddnt |d| dkrz| ritj ||| d|}ntj ||| d}tj |||dntj |||d}tj |||dd ~t ||}t |||~| rt ||}t|dt|t||~t|| }t||t||t |dt|t||t||q|d}g}g}tt|D])}t|d|||| |d}||tdtd|||d}||qt||t||qdS) Nrz#_foreach ops don't support autogradg?cpurrZr)value)r<rZ"_group_tensors_by_device_and_dtypeitemsrZ _foreach_negr9r Zis_cpuZ _foreach_add_rZ _foreach_addZ_foreach_addcmul_Z _foreach_subZ_foreach_maximum_Z_foreach_reciprocal_Z_foreach_copy_Z _foreach_powZ _foreach_mul_r]rangerrDr`)r.rGrIrHrJrKr%r$r'r&r(r r!r"rLZgrouped_tensorsrrXZgrouped_paramsZ grouped_gradsZ grouped_axsZ grouped_musZ grouped_etasZgrouped_state_stepsZ intermediateZnew_musZnew_etasr2rarcrdrrrrV sj                     rVrP)NFFFF)rrZ optimizerrrrrrr r r r Z torch._utilsr typingrr__all__rr__doc__rTr=rrWrVrrrrs ,    !      9      9