o H&i@snddlZddlZddlmZmZmZddlZgdZdZ ddZ ddZ dd Z d d Z d eejeejfd dfddZde ddfddZdde ddfddd eejdeeejeejfdededeeejjdeeejd dfddZ d d eejded dfddZ  d!d eejdeejd dfddZe ddfd eejdeejded dfddZdS)"N)OptionalSequenceUnion) all_reducereduce broadcast all_gatherreduce_scattercCstttjds tddSt}|D]%}|jrdS|s!dS|js'dS| }||vr2dS| |qdS)N_nccl_all_reducez)PyTorch is not compiled with NCCL supportFT) hasattrtorch_CwarningswarnsetZ is_sparseZ is_contiguousZis_cudaZ get_deviceadd)ZtensorsZdevicesZtensorZdevicer:C:\wamp64\www\opt\env\Lib\site-packages\torch/cuda/nccl.py is_available s   rcCsTtj}|d?}|d?d@}|d@}tjd}|dkr$|||fS||||fS)N izutf-8)r r Z _nccl_versionZ_nccl_version_suffixdecode)vermajorminorpatchsuffixrrrversion"s    rcCs tjSN)r r Z_nccl_unique_idrrrr unique_id.s r cCstj|||Sr)r r Z_nccl_init_rank)Z num_ranksuidZrankrrr init_rank2sr"inputsreturncCs&t|tjjr t|tjrtddS)Nz(Inputs should be a collection of tensors) isinstance collectionsabc Containerr Tensor TypeError)r#rrr_check_sequence_type6s r+cCs4t||dur |}t|tj|||||dSr)r+r r r r#outputsopstreamscommsrrrr=s r)r-outputrootr.r/r-cCst||dur|durtdtd||}n!t|tjs1t|tjj r1td||}n |dur9||n|}tj ||||||dS)Nz'output' and 'outputs' can not be both specified. 'outputs' is deprecated in favor of 'output', taking in a single output tensor. The signature of reduce is: reduce(inputs, output=None, root=0, op=SUM, streams=None, comms=None).znccl.reduce with an output tensor list is deprecated. Please specify a single output tensor with argument 'output' instead instead.z\nccl.reduce with an output tensor list is deprecated. Please specify a single output tensor.) r+ ValueErrorrrr%r r)r&r'rr Z _nccl_reduce)r#r1r2r.r/r0r-_outputrrrrGs&   rcCst|tj||||dSr)r+r r Z_nccl_broadcast)r#r2r/r0rrrrnsrcCs&t|t|tj||||dSr)r+r r Z_nccl_all_gather)r#r-r/r0rrrrusrcCs(t|t|tj|||||dSr)r+r r Z_nccl_reduce_scatterr,rrrr sr )rNN)NN)r&rtypingrrrZ torch.cudar __all__ZSUMrrr r"r)r+rintcudaZStreamrrrr rrrrs~ "   (