o 0 h @s ddlZddlZddlZddlZddlZddlZddlmZmZddlm Z ddl m Z ddl m Z mZddlmZddlmZmZmZmZmZmZddlmZddlZddlmZddlmZddl mm!Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z+dd l,m-Z-dd l.m/Z/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6ddl7m8Z8ddl9m:Z:m;Z;mZ>m?Z?m@Z@ddlAmBZBmCZCmDZDmEZEddlmFZFmGZGddlHmIZJddlKmLZLmMZMmNZNmOZOddlPmQZQmRZRmSZSmTZTmUZUddlVmWZWdZXeSrdZYdZZej[\ZXneTrdZYdZZneUrdZYdZZej]\ZXnd ZYd!ZZd"ZXGd#d$d$eZ^Gd%d&d&eZ_Gd'd(d(ej`eZad)ej`d*ejbd+efd,d-Zcd.d/Zd 0 1dd)ej`d2eefd3d4Zfdd5d6Zgd7d8Zhd9d:Zidd)ej`d;eefdeefd?d@Zkd)ej`dAeefdBdCZlGdDdEdEZmGdFdGdGeaZnGdHdIdIeaZoGdJdKdKeoZpGdLdMdMeoZqGdNdOdOeaZrGdPdQdQerZsGdRdSdSej`ZtGdTdUdUeoZuGdVdWdWej`ZvGdXdYdYejwZxGdZd[d[ej`Zyejzd\efd]d^Z{ejzd_efd`daZ|ejzdbefdcddZ}eejzdeefdfdgZ~eejzdhefdidjZeejzdkefdldmZeejzdnefdodpZdqed+edredsefdtduZ vddwej`dxej`dyeedzffd{d|ZGd}d~d~eMZGdddeLZddeefddZGdddej`ZGdddej`ZGdddej`ZdS)N)ABCabstractmethod) nullcontext)deepcopy)autoEnumwraps)AnyCallablecast no_type_checkOptionalUnion)mock) checkpoint) DeviceMesh) CPUOffload fully_shardFullyShardedDataParallel) TrainingState)FSDPParamGroupRegisterPostBackwardFunction)#NO_RESHARD_AFTER_FORWARD_STRATEGIES)BackwardPrefetchMixedPrecisionShardingStrategy)ShardedGradScaler)always_wrap_policyModuleWrapPolicywrap)distribute_tensorDTensorShard)ColwiseParallelparallelize_moduleRowwiseParallelSequenceParallel)TransformerDecoderLayerTransformerEncoderLayer)DistributedDataParallel)MultiProcessTestCaseMultiThreadedTestCase run_subtests TEST_SKIPS) FILE_SCHEMAget_cycles_per_ms TEST_CUDATEST_HPUTEST_XPU) has_tritoncudancclzhpu:0hcclxpuxcclcpuglooc@eZdZeZeZdS) FSDPInitModeN)__name__ __module__ __qualname__rNO_FSDP RECURSIVErErEW/var/www/vscode/kcb/lib/python3.10/site-packages/torch/testing/_internal/common_fsdp.pyr?Vs r?c@seZdZeZeZeZdS)DEVICEInitModeN)r@rArBr DEVICE_BEFORE DEVICE_AFTER DEVICE_NEVERrErErErFrG_s rGc@sneZdZdZedeejdffddZedejfddZ edd d Z e ed e d e de jfd dZdS) FSDPTestModelzZThis defines the interface expected from all models used commonly for FSDP unit tests.return.cCdS)z+Returns an input for the model as as tuple.NrEselfdevicerErErF get_inputlzFSDPTestModel.get_inputcCrM)z,Returns the loss given the input and output.NrE)rOinputoutputrErErFget_lossqrRzFSDPTestModel.get_lossNcCrM)zz)_assert_module_states..cSrhrEri)rk buffer_namebufferrErErFrnrocSsg|]}dqSNrE)rk_rErErFrnsgrouprNr=)named_parameters named_buffersdistget_world_sizerangeall_gather_objectzip) rerfrgnamed_module_states world_sizeolist rank0_statesstatersp1p2rErErF_assert_module_statess"     rcCs ttSrr)r`rP DEVICE_TYPErErErErF get_devtype rFT zero_buffersc Cs|rt|nt}|Q|D]}t |Wdn1s&wYq|rM|D]"}t |Wdn1sGwYq2WddSWddS1s`wYdS)zBZeros the parameters and optionally buffers of ``model`` in place.N)FSDPsummon_full_paramsr parametersr`no_gradzero_buffers)rer summon_fullctxrmrqrErErF _zero_models"      "rcCs"|s|t}|r ||Srr)torhalf state_dict)re cpu_offloadrrErErF_get_state_dicts  rcsdfdd|DS)Nrscs$g|]}|durt|ndqS)Nnone)str)rkstest_name_mappingrErFrns$z subtest_name..)join)rrYrErrF subtest_namesrcCs|D]\}}|jtdkr|||<q|dkr|ndg}t|ttttj f|d}| D] }|| t ||<q5|S)Nr;r) itemsrPr`r;rxbroadcast_object_listr dictrrakeysrr)rankrrlrmrrErErF_broadcast_state_dicts   rrecursecCsBtj||dtt|WdS1swYdS)a[ Returns the full unsharded parameters of ``model``. Any FSDP-managed parameters offloaded to CPU are moved to GPU in the returned list. Args: recurse (bool): If ``False``, only unshards the parameters immediate to ``model``; if ``True``, recurses through the module hierarchy rooted at ``model``. )rN)rrrlistr)rerrErErFget_full_paramss $rmove_to_devicecCs|r|tS|Srr)rr)rerrErErF_move_to_devicer wrap_fsdpcOs|s|St|g|Ri|Srrr)rerrYrZrErErF_maybe_wrap_fsdpsrc@sBeZdZdedefddZdefddZdefdd Zd d Zd S) DummyProcessGrouprsizecCs||_||_dSrr)_rank_size)rOrrrErErF__init__s zDummyProcessGroup.__init__rLcC|jSrr)rrOrErErFrzDummyProcessGroup.rankcCrrr)rrrErErFrrzDummyProcessGroup.sizecOst}dd}||_|S)NcSstj}|d|S)Nr=)r`futuresFuture set_result)futurerErErF get_futures  z/DummyProcessGroup.allreduce..get_future)rMockr)rOrYrZ dist_waitrrErErF allreduceszDummyProcessGroup.allreduceN)r@rArBintrrrrrErErErFrs  rcseZdZdejdededeffdd ZddZd d Z d d Z d dZ e   ddejde dedeeeefdededeejeffddZddZZS)TransformerWithSharedParamsrudevice_init_modeadd_bn deterministiccst||_||_|rtdd}d}t|||_ tj |ddddd|_ t |||_ |j j|j _|d|j j|f|d tj|jtjd d|_|r^tj|jntj|_|tjkrn|t}|rv|dSdS) Nrg?)d_modelnum_encoder_layersnum_decoder_layersdim_feedforwarddropout vocab_bias long_buffer)dtype)superrrrr~r` manual_seedrc Embedding embed_tokens Transformer transformerLinear output_projweightregister_buffernew_ones zeros_likerlongbs BatchNorm1dIdentitybnrGrHrreval)rOrurrrd_vocabr __class__rErFrs>        z$TransformerWithSharedParams.__init__cCsNtd|jtjd|dd|j}tj|jd|dd|j}||fS)Nr= rPr5)r`rrarangeviewr)rOrPsrctgtrErErFrQ.sz%TransformerWithSharedParams.get_inputcCsJ||}||j|j|}||}||}|||}||Srr)rrrtype_asrrr)rOsrc_idstgt_idsrrxrErErFforward4s     z#TransformerWithSharedParams.forwardcCs.|\}}tjj|d|d|dddS)Nsum) reduction)rc functional cross_entropyrr)rOrSrTrsrrErErFrU<sz$TransformerWithSharedParams.get_losscC |dSrrbackwardrVrErErFrXB z(TransformerWithSharedParams.run_backwardNFTfsdp_init_mode fsdp_kwargsrLc Cs|duri}|tjkrt|tr|d}n|}t||||S|tjkrud|vr.ttth}n| d}d|vrI|dt j t j hvrIt|tsId}n|}t|trU|d} n|} t| |||} t | |fd|i|} |tjkrs| t} | Std|)au Initializes a :class:`TransformerWithSharedParams` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps with top-level FSDP. By default, the top-level FSDP uses the ``ModuleWrapPolicy`` for encoder and decoder layers, but a different auto wrap policy may be specified via ``fsdp_kwargs``. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. add_bn (bool): Whether to include batch norm in the model. Nrauto_wrap_policysharding_strategyUnsupported FSDP init mode: )r?rC isinstancer_rrDrr)r(popr HYBRID_SHARD_HYBRID_SHARD_ZERO2rrGrIrr ValueError) rurrrrrpgrfsdp_pg tformer_pgm fsdp_modelrErErFr\EsV          z TransformerWithSharedParams.initcCs|jgSrr)rrrErErFget_ignored_modulessz/TransformerWithSharedParams.get_ignored_modules)NFT)r@rArBrx ProcessGrouprGboolrrQrrUrXrbr?rrrr rrcrdrr\r __classcell__rErErrFrsD*  MrcseZdZdejdededeffdd ZddZd d Z d d Z d dZ e  ddejde dedeeeefdedejf ddZZS)NestedWrappedModulerurrrc st|_|_|tjk}fdd}|r#tdt t t dd||t |t t dd|t t dd||t t dd|t t dd||_ dS)Ncr t|fiS|SrrrlayerrrurrErF _maybe_wrapz1NestedWrappedModule.__init__.._maybe_wraprrr5r)rrrrr~rGrHr`rrc SequentialrrmodulerOrurrrrrrrrrFrs$      zNestedWrappedModule.__init__cCs"td|jtjdd|dfS)Nr=r5rr)r`rrrandrNrErErFrQszNestedWrappedModule.get_inputcC ||SrrrrOrrErErFrrzNestedWrappedModule.forwardcCs |}|Srr)rrOrSrTrWrErErFrUszNestedWrappedModule.get_losscCrrrrrVrErErFrXrz NestedWrappedModule.run_backwardNFrrrLcCsn|duri}|tjkrt|d||dS|tjkr0t|fd||d|}|tjkr.|t}|Std|)a Initializes a :class:`NestedWrappedModule` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. NFrrrTr) r?rCr rDrGrIrrr)rurrrrrrErErFr\s.    zNestedWrappedModule.initNF)r@rArBrxrr rGrrQrrUrXrbr?rrrr rcrdr\r rErErrFr s< r c sJeZdZe  d dejdededee e e fde f fdd Z ZS) AlwaysWrapNestedWrappedModuleNFrurrrrcsltttj|tj|||d}|tjkr|S|tjkr4|pi}t|fdti|}|tj kr2| t }|SdS)z Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, for the ``RECURSIVE`` init mode, this wraps with top-level FSDP and the ``always_wrap_policy()`` auto wrap policy. )rurrrrrN) rrr\r?rCrDrrrGrIrr)rurrrrrerrrErFr\s&    z"AlwaysWrapNestedWrappedModule.initr)r@rArBrbrxrr?rGrrrr r r\r rErErrFrsrc steZdZdejdededeffdd Zedd d Z e  ddejd e ded e e e efdef ddZZS)NonUniformReqGradNWMrurrrc stt||_|_|tjk}fdd}|r%t dt t t dd||t |t t dd|t t dd||t t t dd|t t dd||_dS)Ncr rrrr rrErFr+rz2NonUniformReqGradNWM.__init__.._maybe_wraprrr5r)rr rrrr~rGrHr`rrcrrrrrrrrFrs,     zNonUniformReqGradNWM.__init__rLNcCs,|D]\}}t||s|dqdSr)rvrematchrequires_grad_)re req_grad_masknprErErF_set_nonuniform_req_gradBs   z-NonUniformReqGradNWM._set_nonuniform_req_gradFrrcCstd}|tjkrt|d||d}t|||S|tjkrC|dur%i}t|fd||d|}|tjkr;| t }t|||St d|)a Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, it wraps a second :class:`torch.nn.Sequential` container to enable the desired non-uniform ``requires_grad`` ``use_orig_params=True`` tests. For both ``RECURSIVE`` and ``NO_FSDP`` init modes, freezes all parameters except the last two to validate ``ShardedGradScaler`` support for ranks with no (non-zero sized) local shards in FSDP ``use_orig_params=True`` mode. zmodule\.2.*\.1.*FrNTr) rcompiler?rCrr$rDrGrIrrr)rurrrrreq_grad_pattern ddp_modelrrErErFr\Hs6       zNonUniformReqGradNWM.initr]r)r@rArBrxrr rGrrbr$r?rrrr r\r rErErrFrs4* rc sveZdZdZdejdedeffdd ZddZd d Z d d Z d dZ e de ededededef ddZZS)ModuleWithDelayzThis class wraps a :class:`FSDPTestModel` to optionally add a delay after computing the loss and/or before the gradient reduction.rdelay_after_loss_msdelay_before_reduction_mscs t||_||_||_dSrr)rrr)r*r)rOrr)r*rrErFr{s  zModuleWithDelay.__init__cCs |j|Srr)rrQrNrErErFrQrzModuleWithDelay.get_inputcCrrrrrrErErFrrzModuleWithDelay.forwardcCsT|j||}|jdkr(tstrt|jd|Str(tj t |jt |SNr) rrUr)r2r3timesleepr1r`r6_sleeprr0rrErErFrUs zModuleWithDelay.get_losscsTtjjfdd}td|j|WddS1s#wYdS)NcsNjdkr trtjtjtn tstr t jd|i|Sr+) r*r1r`r6r/rr0r2r3r-r.r[orig_reduce_scatterrOrErF_delayed_reduce_scatters z=ModuleWithDelay.run_backward.._delayed_reduce_scatterz'torch.distributed.reduce_scatter_tensor)r` distributedreduce_scatter_tensorrpatchrrX)rOrWr2rEr0rFrXs "zModuleWithDelay.run_backward module_class model_args model_kwargscOst|j|i|||S)aA Args: module_class (Type[FSDPTestModel]): Wrapped module class to which to add delays. model_args: Positional arguments forwarded to the ``module_class`` ``init()``. delay_after_loss_ms (int): Delay after computing the loss/before the optimizer step (in ms). delay_before_reduction_ms (int): Delay before reduce-scattering gradients (in ms). model_kwargs: Keyword arguments forwarded to the ``module_class`` ``init()``. )r(r\)r6r)r*r7r8rErErFr\s zModuleWithDelay.init)r@rArBr^rcrdrrrQrrUrXrbtyperKr r\r rErErrFr(ws2  r(c@sReZdZeejddddfdejdedede e e e fde d ed efd d ZdS) NestedWrappedModuleWithDelayNFrrurrrrr)r*c Cstjt|||||||dS)Nrurrrrr)r*)r(r\r r;rErErFr\s z!NestedWrappedModuleWithDelay.init)r@rArBrbrGrIrxrr?rrrr r rr\rErErErFr:s,r:c$eZdZfddZddZZS)DummyDDPcst||_dSrr)rrr)rOrrrErFrs  zDummyDDP.__init__cOs|j|i|SrrrrOrYrZrErErFrszDummyDDP.forwardr@rArBrrr rErErrFr=s r=cseZdZdejdedededef fdd Zdd Z d d Z e ddejde dede eeefdedef ddZZS)MixtureOfExpertsrurrdelay_before_free_msrc s$tj||||d||_||_||_|tjk|_|r#t d|j d}d}d} t t |||j} tdd| D|_| D]} d| _qC|rPt d t t |||j} |rwtj| g} t| | fi|} t| |fi|} t t t | ||j| | t t || |j|_dS) N)rurrr*rrrcs|]}|VqdSrr)numel)rkr#rErErF z,MixtureOfExperts.__init__..Tr)rrrurArrGrHrr`rrrrcrrrnum_expert_paramsexpertr3 new_grouprrr)rOrurrrArrd_expertd_sharedd_inputrHr#shared expert_grouprrErFrsD     zMixtureOfExperts.__init__csxjdkr7jd}t|tr7tjjjjfdd}t d| |WdS1s2wY|S)NrrcsDtrtjtjtn tstrt jd|i|S)Nr,) r1r`r6r/rrAr0r2r3r-r.r[ orig_reshardrOrErF_delayed_reshardsz2MixtureOfExperts.forward.._delayed_reshardz.torch.distributed.fsdp._runtime_utils._reshard) rArrrr`r3fsdp_runtime_utils_reshardrr5)rOrrHrQrErOrFrs      zMixtureOfExperts.forwardcCs||jsAt,|D]}t|drq|jdur.|j|jtj j |j|j dqWddS1s:wYdSdS)NrHrt) rrr`rrhasattrgraddiv_r~r3 all_reduceru)rOrWr#rErErFrX1s    "zMixtureOfExperts.run_backwardNFrrrcCsr|duri}|tjkrt|d|||dS|tjkr2t|fd|||d|}|tjkr0|t}|Std|)a Initializes a :class:`MixtureOfExperts` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP, including the expert and shared layers, but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. delay_before_free_ms (int): Delay before resharding expert parameters in the forward pass (in ms). NF)rrrArTr) r?rCr@rDrGrIrrr)rurrrrrArrErErFr\=s2    zMixtureOfExperts.init)NFr)r@rArBrxrr rGrrrrXrbr?rrrr r\r rErErrFr@s>4 r@c sdeZdZ ddddddedeejded ed ef fd d Zd ej dej fddZ ddZ Z S)MLPNTFr5)bias with_bufferdim_multiplierdimrPrZr[r\csdttj|||||d|_tj|||||d|_|r-|dtj|f|ddSd|_ dS)N)rPrZrqr) rrrcrin_projout_projrr`randnrq)rOr]rPrZr[r\rrErFrrs  z MLP.__init__rrLcCs@||}t|}||}t|}|jdur||j}|Srr)r^Frelur_rq)rOrzrErErFrs      z MLP.forwardcCs"|jdurtjj|jdSdSrr)rqr`rcr\normal_rrErErFreset_parameterss zMLP.reset_parametersrr) r@rArBrrr`rPr rrarrer rErErrFrYqs& rYcsFeZdZdddedeffddZdeded ed dfd d ZZS) MLPStackF)with_seq_parallelmlp_dimrgcsLt|ddt|t|ddg}|r|tj|ddtj|||_dS)N)r\FrZ)rYappendrc LayerNormrrrg)rOrhrgmodulesrrErFrs    zMLPStack.__init__tp_meshdp_meshuse_activation_checkpointingrLcKstddtddtddtddtdd|jrttddntd}|jr-tdd|d<t|||d|D]}t|tjr?q6|rEt |t |fd |i|q6t |fd |i||S) NF)use_local_outputr=)output_layouts)z 0.in_projz 0.out_projz 1.in_projz 1.out_projz 2.in_projz 2.out_proj) sequence_dim3) device_meshparallelize_planmesh) r$r&rgr#r'r%rrcrlrr)rOrnrorprrvrrErErF parallelizes(   zMLPStack.parallelize) r@rArBrr rrrxr rErErrFrfs rfcsVeZdZdZd dedeffdd Zdejde e ejejfejffd d Z Z S) DoubleLinearz This can be used for returning multiple outputs from a module (``use_second_linear=True``) or for having an unused module (``False``). Tr]use_second_linearcs:tt|||_t|||_t|_||_dSrr) rrrcrlin1lin2ReLUrbrz)rOr]rzrrErFrs   zDoubleLinear.__init__rrLcCs6|jr||||||fS|||Srr)rzrbr{r|rrErErFrs zDoubleLinear.forwardT) r@rArBr^rr rr`rarr_rr rErErrFrysrynew_all_gather_into_tensorccBtj}t|t_z dVWt|t_dSt|t_wrr)rxall_gather_into_tensorbarrier)rorig_all_gatherrErErFpatch_all_gather rnew_reduce_scatter_tensorccrrr)rxr4r)rr1rErErFpatch_reduce_scatterrrnew_all_reduceccrrr)rxrXr)rorig_all_reducerErErFpatch_all_reducerr new_unshardccBtj}t|t_z dVWt|t_dSt|t_wrr)runshardrxr)r orig_unshardrErErF patch_unshard r new_reshardccrrr)rreshardrxr)rrPrErErF patch_reshard rrnew_post_backwardccrrr)r post_backwardrxr)rorig_post_backwardrErErFpatch_post_backwardrr new_backwardccrrr)rrrxr)r orig_backwardrErErF*patch_register_post_backward_hook_backward$rrr1rYrZcOsRt|dkr |d}nd|vr|d}n td|d|||||i|S)NrrTz,Cannot get reduce-scatter output from args: z kwargs: )lenAssertionError)clsr1rgrYrZrTrErErFreduce_scatter_with_assert1s   rrEreplicated_modulesharded_moduleprefixes_to_ignore.cCst||D]\\}}\}}|}|D]} || d}q|||||tt|ts1J|j|j} } t | t dt dfkrHt dt || | } || | |jdurd||jq ||jt |j| | } ||jtt|jtsJ||j | q dS)NrzmFSDP's (Shard(0), Shard(0)) layout differs from distribute_tensor(), so we cannot check for equality using it)r|rvreplace assertEqualassertIsInstancer"rru placementsr_r#rr!to_localrV assertIsNoneassertIsNotNone)rrrrreplicated_namereplicated_param sharded_name sharded_paramclean_sharded_nameprefixrwrsharded_ref_paramsharded_ref_gradrErErFcheck_sharded_parityDs2       rcs@eZdZeddZfddZddZddZd d ZZ S) FSDPTestMultiThreadcCtSrr DEVICE_COUNTrrErErFr~fzFSDPTestMultiThread.world_sizecst|dSrr)rsetUp_spawn_threadsrrrErFrjs  zFSDPTestMultiThread.setUpcOt|g|Ri|Srrr-r>rErErFr-nz FSDPTestMultiThread.run_subtestscCtjdSrrr`_dynamoresetrrErErFperThreadSetUpqz"FSDPTestMultiThread.perThreadSetUpcCrrrrrrErErFperThreadTearDowntrz%FSDPTestMultiThread.perThreadTearDown) r@rArBpropertyr~rr-rrr rErErrFres  rc$sjeZdZfddZeddZeddZedefdd Zed d Z d d Z ddZ ddZ ddZ eddZ       d4dejdedededeededeed ed!ed"eeeeffd#d$Zdd%d&edddddddddf d'eed(ed)ed*eed+eded,ed-ee d.ee!deed/ed0ed ed!ed1eeeefd"eeeeff d2d3Z"Z#S)5FSDPTestcs tdtjd<|dS)N0TORCH_NCCL_DESYNC_DEBUG)rrosenviron_spawn_processesrrrErFrys   zFSDPTest.setUpcCrrrrrrErErFr~rzFSDPTest.world_sizecCs tjSrr)rxdistributed_c10d_get_default_grouprrErErFrfs zFSDPTest.process_grouprLcCrMrrErrErErFdestroy_pg_upon_exitrRzFSDPTest.destroy_pg_upon_exitcCst|jSrr)r/ file_namerrErErF init_methodszFSDPTest.init_methodcC|||jdSrr)rr)rOrrrErErF_check_cpu_offloadrzFSDPTest._check_cpu_offloadcCrrr)rbackward_prefetch)rOrrrErErF_check_backward_prefetchrz!FSDPTest._check_backward_prefetchcCrrr)rforward_prefetch)rOrrrErErF_check_forward_prefetchrz FSDPTest._check_forward_prefetchcOrrrrr>rErErFr-rzFSDPTest.run_subtestsc Ks(||}||_||_|dd}td|jd|jz%|r2tjjjj }t j d|j||dnt j |j tt|j|jdWnty]} zd| jd vrXttd jd} ~ wwd} |jt} tsitrotj| | g} t j| d tj|||tjt j| d t dS) Nfake_pgFz dist init r=z, world=fake)backendr~rstore)rrr~r recompilerbackend_unavailable) device_ids) rrgetprintr~r`testing _internalr3r FakeStorerxinit_process_grouprDISTRIBUTED_BACKENDr RuntimeErrorrYsysexitr. exit_coderr1r3 acceleratorset_device_indexrrrrun_testdestroy_process_group) rr test_namerpiperZrOrrer device_idrErErF_runsL         z FSDPTest._run{Gz?NFre num_stepsautocastlrfsdp_cpu_offload save_modelmixed_precisionenable_sharded_grad_scaler use_pure_fp16sharded_grad_scaler_kwargsc  CsF|o|j} t|j} | duri} td d|i| } tjj||dd}t|D]}| tj j t |dY|j tt }| sK|r_t|ts_t|tjrV|}n tdd|D}||}| rt|tr|jtvr|D] }||jtdqs|j ||| }Wdn1swY| |}|s| s|jtjksJdn'| r||jtjnt|tr|dusJ||j|jn||jtj|j || rt|tr|D] }||jtdq| || |rd d |!"D}t#||$|q*t|tr|%t&j'|(S) Nenabledg?)rmomentum)rcsrCrr)r)rkrrErErFrErFz4FSDPTest._train_for_several_steps..r;zeloss data type should be float32, as the original parameter data type is float32.cSsi|] \}}||qSrE)clone)rkkvrErErF sz5FSDPTest._train_for_several_steps..rE))offload_paramsnextrrPrr`optimSGDrz zero_gradamprrrrQrrrarr_rrrrUrscalerfloat32float16 param_dtyperXstepupdaterrrload_state_dict _assert_staterIDLErj)rOrerrrrrrrrrcpu_offload_params model_devicesharded_grad_scalerrrsrSrTr#rWrrErErF_train_for_several_stepssn              z!FSDPTest._train_for_several_stepsrT model_classrr ref_init_fn num_itersrrrruse_orig_params init_kwargsc" Ks|tjks Jd|duri}d}|j}|j|jtjtjfddi|}|dur>tr5t|t gt d}n t||g|d}n||}|rH| }|j ||| du||| | ||d }t | }|||| | | | dz|j|j|||fddi|}Wnty}z td |d t||d}~wwt|tst||jfi|}|r| }|tjkr|t }|duo|j}|o|tjk}|o|tjk}|rtd }| D] }||j|q|r|td t nt}||j ||d |||| | ||d } Wdn 1s wY|rdS|r/td }| D] }||j|q| t } t|}!tjj|| d d| durO|sQ|j||!ddddSdSdS)a Tests FSDP training against a reference, which defaults to DDP but may be customized with ``ref_init_fn``. Args: model_class (Type[FSDPTestModel]): A model class that inherits from ``FSDPTestModel``, which defines the expected interface. fsdp_init_mode (FSDPInitMode): The mode to initialize the FSDP-wrapped model. This should not be ``NO_FSDP``. ref_init_fn (Optional[Callable]): A callable to invoke that wraps a non-wrapped model to construct the reference model, where this wrapper should provide data parallel semantics. If ``None``, then the callable defaults to the DDP constructor. z.Expects an FSDP init mode that wraps with FSDPNrrT)r output_device)rrrrrrr)rrrrrrz Initializing z raised error r;zOAn FSDP-managed module with parameter CPU offloading enabled has parameters on F)rrrrrrrr) check_dtypezFSDP did not match DDP) exact_devicemsg) r?rCrfrr\rGrHr2DDPrrrrrr  ExceptionrrrrrIrrr`rPrassertRaisesRegexrrrr assert_close)"rOrrrrrrrrrrrrrrrrrrrre ref_modelref_loss ddp_paramsrrrexpects_device_errorexpects_cpu_device cpu_devicermcontext fsdp_lossfsdp_unsharded_paramsrErErF_test_fsdp_parity)s #                zFSDPTest._test_fsdp_parity)rNFNFFN)$r@rArBrrr~rfr rrrrrr- classmethodrrcrdrfloatrrrrrr rr9rKr?rGr rrr*r rErErrFrxs     8   \     rcompile_compute_on_modulecs.fddGdddtfdd}|S)Ncs>tjjj|i|dust|dr|ddSdS)Nr)r`r3rRrrr%r[)r-rErF!fully_shard_with_compiled_computes  z=compiled_fsdp_test..fully_shard_with_compiled_computec@r>)z*compiled_fsdp_test..FullyShardModeN)r@rArBrEAGERCOMPILED_COMPUTErErErErFFullyShardModes r1cstfdd}|S)Ncstjjj}D]b}|jkrtstdqtjj j }tj j j }tj |jkr.|}n|jkr@dtjj _ dtj j _ }ntd||j|j<|i|tj |j|j<|tjj _ |tj j _ qdS)Nz0Inductor on GPU needs Triton and recent GPU archTr=z!Need to implement FullyShardMode=)r`r3rRrr/r4warningswarnrconfigskip_fsdp_hooks _inductorcompile_threadsrr0NotImplementedError __globals__r@)rYrZoriginal_fully_shardmodeoriginal_skip_fsdp_hooksoriginal_compile_threadsfully_shard_patch)r1r.funcrErFwrappers0              z6compiled_fsdp_test..decorator..wrapperr)r?r@)r1r.)r?rF decorators z%compiled_fsdp_test..decorator)r)r-rArE)r1r-r.rFcompiled_fsdp_tests $rBcs&eZdZdfdd ZddZZS) SkipModulerLNcs ttjdddd|_dSN Frj)rrrcrlinrrrErFrs zSkipModule.__init__cCrrr)rFrrErErFr rzSkipModule.forwardr]r?rErErrFrCsrCcr<) NestedLinearcsJt|rttjddddt|_dStjddddt|_dSrD)rrr rcrrr nested_linear)rO fsdp_wraprrErFrs  zNestedLinear.__init__cCrrr)rHrrErErFrrzNestedLinear.forwardr?rErErrFrGs rGcr<) SkipModelcsHttjddddt|_tt|_t t |dtd|_ dS)NrEFrj)rI)r) rrrcrrrlinearrC linear_skipr rGrH)rO double_nestrrErFrs   zSkipModel.__init__cCs"||}||}||}|Srr)rKrLrHrrErErFr$s   zSkipModel.forwardr?rErErrFrJs rJ)FT)FFr~)rErr) contextlibrrrr-r2abcrrrcopyrenumrr functoolsr typingr r r r rrunittestrr`torch.distributedr3rxtorch.nnrctorch.nn.functionalrratorch.distributed._composablertorch.distributed.device_meshrtorch.distributed.fsdprrrr$torch.distributed.fsdp._common_utilsr5torch.distributed.fsdp._fully_shard._fsdp_param_grouprr"torch.distributed.fsdp._init_utilsr2torch.distributed.fsdp.fully_sharded_data_parallelrrr*torch.distributed.fsdp.sharded_grad_scalerrtorch.distributed.fsdp.wraprrr torch.distributed.tensorr!r"r#!torch.distributed.tensor.parallelr$r%r&r'r(r)torch.nn.parallel.distributedr*r*torch.testing._internal.common_distributedr+r,r-r.$torch.testing._internal.common_utilsr/r0r1r2r3torch.utils._tritonr4rrrr6 device_countr9r?rGrdrKrrrr rrrrrrrrrr rrr(r:r=r@rYrrfrycontextmanagerrrrrrrrrr_rrrrr9rBrCrGrJrErErErFs                     ^"`M  -          !]3