o V h@sddlZddlmZddlmZddlmZmZmZm Z m Z m Z m Z ddl Z ddlZ ddlmZddlmZmZddlmZddlmZd d lmZmZmZd d lmZd d lmZm Z gd Z!eGdddZ"de e#de#fddZ$de j%de#de#de e j%e#ffddZ&de j%de#de#de#de j%f ddZ'e j()de j()dGdddej*Z+de j%de#de j%fd d!Z,d"e j%d#e j%d$e e#e#e#fd%e e#e#e#fd&e j%d'e j%d(e j%de j%fd)d*Z-de j%d+e j%d,e.fd-d.Z/e j()d*e j()d.Gd/d0d0ej*Z0Gd1d2d2ej*Z1Gd3d4d4ej*Z2Gd5d6d6ej*Z3d7e e"d8e4d9e ed:e.d;ede3f dd?d?eZ6Gd@dAdAeZ7ee dBe6j8fdCddDdEd9e e6d:e.d;ede3fdFdGZ9ee dBe7j8fdCddDdEd9e e7d:e.d;ede3fdHdIZ:dS)JN) dataclass)partial)AnyCallableDictListOptionalSequenceTuple)MLPStochasticDepth)VideoClassification)_log_api_usage_once)register_modelWeights WeightsEnum)_KINETICS400_CATEGORIES)_ovewrite_named_paramhandle_legacy_interface)MViTMViT_V1_B_Weights mvit_v1_bMViT_V2_S_Weights mvit_v2_sc@sVeZdZUeed<eed<eed<eeed<eeed<eeed<eeed<dS) MSBlockConfig num_headsinput_channelsoutput_channelskernel_q kernel_kvstride_q stride_kvN)__name__ __module__ __qualname__int__annotations__rr)r)Q/var/www/vscode/kcb/lib/python3.10/site-packages/torchvision/models/video/mvit.pyrs    rsreturncCsd}|D]}||9}q|SNr))r+productvr)r)r*_prod&s r1x target_dim expand_dimcCsF|}||dkr||}||fS||krtd|j||fS)Nr.zUnsupported input dimension )dim unsqueeze ValueErrorshaper2r3r4 tensor_dimr)r)r* _unsqueeze-s  r;r:cCs||dkr ||}|Sr-)squeezer9r)r)r*_squeeze6s  r=c s|eZdZ  ddejdeejdeejdeddf fdd Zd ej d e e e e fde ej e e e e fffd d Z Z S)PoolNFpoolnorm activationnorm_before_poolr,csVt||_g}|dur|||dur|||r#tj|nd|_||_dS)N)super__init__r?appendnn Sequentialnorm_actrB)selfr?r@rArBlayers __class__r)r*rDAs    z Pool.__init__r2thwc Cst|dd\}}tj|ddd\}}|dd}|jdd\}}}||||f|}|jr<|jdur<||}| |}|jdd\}} } ||||ddd}tj ||fdd}|jsm|jdurm||}t |dd|}||| | ffS) Nr.r.r)indicesr5r r5) r;torch tensor_split transposer8reshape contiguousrBrHr?catr=) rIr2rMr: class_tokenBNCTHWr)r)r*forwardRs    z Pool.forward)NF)r$r%r&rFModulerboolrDrSTensorr r'r` __classcell__r)r)rKr*r>@s>r> embeddingdcCs@|jd|kr |Stjj|ddd|dddddS)Nrr.linear)sizemode)r8rF functional interpolatepermuter6r<)rerfr)r)r* _interpolatelsrmattnqq_thwk_thw rel_pos_h rel_pos_w rel_pos_tc% Cs6|\}}} |\} } } tdt|| d} tdt| | d}tdt|| d}t| |d}t|| d}t|dddf|t| dddfd| |}t| | d}t| | d}t| dddf|t| dddfd| |}t| |d}t|| d}t|dddf|t| dddfd| |}t|| }t||}t||}||}||}||}|j\}}}}|ddddddf||||| |} td| |}!td| |}"| dddddd ||||| |} t | | dd dd}#|# |||| ||  dddddd }#|!ddddddddddddddf|"ddddddddddddddf|#ddddddddddddddf||||| | | | }$|ddddddddf|$7<|S) Nrr.?zbythwc,hkc->bythwkzbythwc,wkc->bythwkrr rN) r'maxrSarangermlongr8rVeinsumrlmatmulrUview)%rnrorprqrrrsrtq_tq_hq_wk_tk_hk_wdhdwdt q_h_ratio k_h_ratiodist_h q_w_ratio k_w_ratiodist_w q_t_ratio k_t_ratiodist_tRhRwRtrZn_head_r5r_qrel_h_qrel_w_qrel_q_trel_posr)r)r* _add_rel_pos{sH <<<      **$...(rshortcutresidual_with_cls_embedc CsZ|r |||S|ddddddddf|ddddddddf7<|Sr-)add_)r2rrr)r)r* _add_shortcuts  DrcseZdZdejfdeededededeedeedeed eed ed ed ed ede dej fddffdd Z de j deeeefdee j eeeefffddZZS)MultiscaleAttention input_size embed_dim output_dimrr r!r"r# residual_poolr rel_pos_embeddropout norm_layer.r,Nc spt||_||_||_|||_dt|j|_| |_ | |_ t |d||_ t ||g}| dkr@|t j| ddt j||_d|_t|dksUt|dkrrdd|D}tt j|j|j||||jd d | |j|_d|_d|_t|dkst|dkrd d|D}tt j|j|j||||jd d | |j|_tt j|j|j||||jd d | |j|_d|_d|_d|_| r6t|dd}t|d kr||dn|}t|d kr||dn|}d t||d}d |d d}t t||j|_t t||j|_t t||j|_t j j!|jddt j j!|jddt j j!|jdddSdS)Nrur rTinplacer.cSg|]}t|dqSrr').0ror)r)r* z0MultiscaleAttention.__init__..F)stridepaddinggroupsbiascSrrr)rkvr)r)r*rrrr{Gz?std)"rCrDrrrhead_dimmathsqrtscalerrrrFLinearqkvrEDropoutrGprojectpool_qr1r>Conv3dpool_kpool_vrrrsrtrwlen ParameterrSzerosinit trunc_normal_)rIrrrrr r!r"r#rrrrrrJ padding_q padding_kvrhq_sizekv_size spatial_dim temporal_dimrKr)r*rDs         zMultiscaleAttention.__init__r2rMc Cs:|j\}}}||||d|j|jddjdd\}}}|jdur-|||\}} n|} |jdur<|||d}|j durI| ||\}}t |j ||dd} |j durr|jdurr|jdurrt| ||| |j |j|j} | jdd} t | |}|jrt|||j|dd|d|j}||}||fS)Nr r.rrRrrQ)r8rrVrrrUunbindrrrrSr{rrrrsrtrsoftmaxrrrrr) rIr2rMrZr[r\rokr0rqrnr)r)r*r` s6 2     zMultiscaleAttention.forward)r$r%r&rF LayerNormrr'rbfloatrrarDrSrcr r`rdr)r)rKr*rsB      >\rcseZdZddejfdeedededededede d e d e d ej fd d ffdd Z de jdeeeefd ee jeeeefffddZZS)MultiscaleBlockrrcnfrrrproj_after_attnrstochastic_depth_probr.r,Nc st||_d|_t|jdkr.dd|jD} dd| D} ttj| |j| dd|_|r3|j n|j } | |j |_ | | |_ t |j tj|_t||j | |j|j|j|j|j||||| d |_t| d| |j gtj|dd|_t|d |_d|_|j |j krt|j |j |_dSdS) Nr.cSs g|] }|dkr |dn|qSrOr))rr+r)r)r*rUs z,MultiscaleBlock.__init__..cSrrr)rrr)r)r*rVr)rr) r r!r"r#rrrrrrN)activation_layerrrrow)rCrDr pool_skipr1r"r>rF MaxPool3drrnorm1norm2 isinstance BatchNorm1dneeds_transposalrrr r!r#rnr GELUmlpr stochastic_depthrr) rIrrrrrrrrr kernel_skip padding_skipattn_dimrKr)r*rDDsP      zMultiscaleBlock.__init__r2rMc Cs|jr||ddddn||}|||\}}|jdus%|js'|n||}|jdur3|n|||d}|||}|jrR||ddddn||}|jdus_|jra|n||}||| ||fS)Nr.rr) rrrUrnrrrrrr) rIr2rMx_norm1x_attnthw_newx_skipx_norm2x_projr)r)r*r`~s**zMultiscaleBlock.forward)r$r%r&rFrrr'rrbrrrarDrSrcr r`rdr)r)rKr*rCs4    >:rc sPeZdZdedeeefdededdf fdd Zd ejdejfd d Z Z S) PositionalEncoding embed_size spatial_size temporal_sizerr,Ncst||_||_tt||_d|_ d|_ d|_ |sGtt|jd|jd||_ tt|j||_ tt||_ dSdS)Nrr.) rCrDrrrFrrSrrY spatial_pos temporal_pos class_pos)rIrrrrrKr)r*rDs $zPositionalEncoding.__init__r2cCs|j|ddd}tj||fdd}|jdur\|jdur\|jdur\|jj \}}tj |j|dd}| |jd|j dd d|tj|jd|fddd}| ||S)NrrQr.rR)rYexpandrhr6rSrXrrrr8repeat_interleaverrrV)rIr2rYhw_sizer pos_embeddingr)r)r*r`s &  zPositionalEncoding.forward) r$r%r&r'r rbrDrSrcr`rdr)r)rKr*rs*rc$seZdZ         ddeeefd ed eed ed ed edededededede e de j fde e de j fdeeeefdeeeefdeeeefddf"fdd Z dejdejfddZZS) r?rNr rrrNrNr.r r rr block_settingrrrrrattention_dropoutr num_classesblock.rpatch_embed_kernelpatch_embed_stridepatch_embed_paddingr,cstt|t|}|dkrtd| durt} | dur&ttjdd} tj d|dj |||d|_ dd t |f||j j D}t|dj |d |d f|d|d |_t|_t|D]/\}}| ||d }|j| ||||||| || d t|jdkrdd t ||jD}q`| |dj|_ttj|ddt|dj| |_|D][}t|tjrtjj|jddt|tjr|j durtj!|j dqt|tjr|jdurtj!|jd |j durtj!|j dqt|tr |"D] }tjj|ddqqdS)a MViT main class. Args: spatial_size (tuple of ints): The spacial size of the input as ``(H, W)``. temporal_size (int): The temporal size ``T`` of the input. block_setting (sequence of MSBlockConfig): The Network structure. residual_pool (bool): If True, use MViTv2 pooling residual connection. residual_with_cls_embed (bool): If True, the addition on the residual connection will include the class embedding. rel_pos_embed (bool): If True, use MViTv2's relative positional embeddings. proj_after_attn (bool): If True, apply the projection after the attention. dropout (float): Dropout rate. Default: 0.0. attention_dropout (float): Attention dropout rate. Default: 0.0. stochastic_depth_prob: (float): Stochastic depth rate. Default: 0.0. num_classes (int): The number of classes. block (callable, optional): Module specifying the layer which consists of the attention and mlp. norm_layer (callable, optional): Module specifying the normalization layer to use. patch_embed_kernel (tuple of ints): The kernel of the convolution that patchifies the input. patch_embed_stride (tuple of ints): The stride of the convolution that patchifies the input. patch_embed_padding (tuple of ints): The padding of the convolution that patchifies the input. rz+The configuration parameter can't be empty.Ngư>)epsr ) in_channels out_channels kernel_sizerrcSg|]\}}||qSr)r)rrhrr)r)r*rrz!MViT.__init__..r.r)rrrrru) rrrrrrrrrcSrr)r)r r)r)r*r rrQTrrrr)#rCrDrrr7rrrFrrr conv_projziprr pos_encoding ModuleListblocks enumeraterEr"rr@rGrrheadmodulesrrrweightr constant_ parameters)rIrrrrrrrrrrrrrrrrtotal_stage_blocksrstage_block_idrsd_probmweightsrKr)r*rDs )         z MViT.__init__r2cCst|ddd}||}|ddd}||}|jjf|jj}|jD] }|||\}}q'||}|dddf}| |}|S)Nrvrrr.) r;r flattenrUr rrrr@r)rIr2rMrr)r)r*r`!s     z MViT.forward) rrrrNNrrr)r$r%r&r r'r rrbrrrrFrarDrSrcr`rdr)r)rKr*rs\          xrrrrprogresskwargsc Ks|dur1t|dt|jd|jdd|jddksJt|d|jdt|d|jd|dd }|dd }td||||d d |d d|dd |dd |d|}|durk||j|dd|S)Nr categoriesmin_sizerr.rrmin_temporal_sizer!rFrTrr)rrrrrrrr)r check_hashr))rrmetapoprload_state_dictget_state_dict)rrrrrrrmodelr)r)r*_mvit8s,         r)c @JeZdZedeedddddddedd d d d d diddd dZeZdS)rz:https://download.pytorch.org/models/mvit_v1_b-dbeb1030.pthr ?r.r.?r0r0 crop_size resize_sizemeanrr"zShttps://github.com/facebookresearch/pytorchvideo/blob/main/docs/source/model_zoo.mdThe weights were ported from the paper. The accuracies are estimated on video-level with parameters `frame_rate=7.5`, `clips_per_video=5`, and `clip_len=16`ip. Kinetics-400gJ +S@gh|?eW@zacc@1zacc@5guVQ@g rxa@ rrrrecipe_docs num_params_metrics_ops _file_sizeurl transformsr$N r$r%r&rrrrKINETICS400_V1DEFAULTr)r)r)r*rY4rc @r*)rz:https://download.pytorch.org/models/mvit_v2_s-ae3be167.pthr r+r-r/r1r"zChttps://github.com/facebookresearch/SlowFast/blob/main/MODEL_ZOO.mdr5ir6g r0T@g(\W@r7guVP@g?5^I|`@r8r?NrBr)r)r)r*rzrEr pretrained)rT)rrcKst|}gdgdgdggdggdgggggggggggdgggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgggdggdgggggggggggdgggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgd }g}tt|d D],}|t|d ||d ||d ||d ||d||d||d|d qtddd|dd|dd||d|S)a Constructs a base MViTV1 architecture from `Multiscale Vision Transformers `__. .. betastatus:: video module Args: weights (:class:`~torchvision.models.video.MViT_V1_B_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.video.MViT_V1_B_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.video.MViT`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.video.MViT_V1_B_Weights :members: r.rrrNrNrNrNrNrNrNrNrNrNrNrH`rKrLrLrLrLrLrLrLrLrLrLrM)rKrKrLrLrLrLrLrLrLrLrLrLrLrMrMrMr r r r.rrr.rHrHr.rNrNr.r.r.rrrr r!r"r#rrrr r!r"r#r r"Fr皙?)rrrrrrrrNr))rverifyrangerrErr)r%rrrconfigrir)r)r*rs ..,          rcKsDt|}gdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdggdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgdgd }g}tt|d D],}|t|d ||d ||d ||d ||d||d||d|d qtddd|dddd|dd||d |S)aCConstructs a small MViTV2 architecture from `Multiscale Vision Transformers `__ and `MViTv2: Improved Multiscale Vision Transformers for Classification and Detection `__. .. betastatus:: video module Args: weights (:class:`~torchvision.models.video.MViT_V2_S_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.video.MViT_V2_S_Weights` below for more details, and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.video.MViT`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.video.MViT_V2_S_Weights :members: rG)rJrJrKrKrLrLrLrLrLrLrLrLrLrLrLrMrIrNrRrOrPrQrSrrrr r!r"r#r r"TFrrT) rrrrrrrrrrNr))rrUrVrrErr)r%rWr)r)r*rs N          r);r dataclassesr functoolsrtypingrrrrrr r rStorch.fxtorch.nnrFopsr r transforms._presetsrutilsr_apirrr_metar_utilsrr__all__rr'r1rcr;r=fxwraprar>rmrrbrrrrrrr)rrrCrrr)r)r)r*s  $      &"  ,   <  H !!!*`.