o I hCT@sdZddlZddlZddlmZddlZddlmZddgZGdddejj Z Gdd d ejj Z Gd d d ejj Z Gd ddejj Z dS) z We will recreate all the RNN modules as we require the modules to be decomposed into its building blocks to be able to observe. N)Optional)TensorLSTMCellLSTMc seZdZdZejjZdgZ   dddde de d e d dffd d Z dd e de ee e fd ee e ffddZ dde de d ee e ffddZddZedddZedddZZS) raA quantizable long short-term memory (LSTM) cell. For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell` `split_gates`: specify True to compute the input/forget/cell/output gates separately to avoid an intermediate tensor which is subsequently chunk'd. This optimization can be beneficial for on-device inference latency. This flag is cascaded down from the parent classes. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTMCell(10, 20) >>> input = torch.randn(6, 10) >>> hx = torch.randn(3, 20) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) split_gatesTNFr input_dim hidden_dimbiasreturnc s||d}t||_||_||_||_|sAtjj|d|fd|i||_ tjj|d|fd|i||_ tj jj |_nAtj|_ tj|_ tj|_dD],}tjj||fd|i||j |<tjj||fd|i||j |<tj jj |j|<qUtj|_tj|_tj|_tj|_tj jj |_tj jj |_tj jj |_tj jj |_d|_d|_tj|_tj|_dS)Ndevicedtyper )inputforgetcelloutput)?r)super__init__ input_size hidden_sizer rtorchnnLinearigateshgatesao quantizedFloatFunctionalgates ModuleDictSigmoid input_gate forget_gateTanh cell_gate output_gatefgate_cx igate_cgatefgate_cx_igate_cgateogate_cyinitial_hidden_state_qparamsinitial_cell_state_qparamsquint8hidden_state_dtypecell_state_dtype) selfrr r r rrfactory_kwargsg __class__W/var/www/vscode/kcb/lib/python3.10/site-packages/torch/ao/nn/quantizable/modules/rnn.pyr,sj           zLSTMCell.__init__xhiddencCsr|dus|ddus|ddur||jd|j}|\}}|jsQ||}||}|j||}|dd\}} } } | |}| | } | | } | | } nAi} t |j|j|jD]\\} }}}|||||| | <qb| | d}| | d} | | d} | | d} |j| |}|j|| }|j||}|}t|}|j| |}||fS)Nrrrrrr)initialize_hiddenshape is_quantizedrrrr!addchunkr$r%r'r(zipitemsvaluesr)mulr*r+rtanhr,)r2r9r:hxcxrrr!r$r%r'out_gategatekeyr)r*r+cytanh_cyhyr7r7r8forwardfs<        zLSTMCell.forward batch_sizer>c Csjt||jft||jf}}|r1|j\}}|j\}}tj||||jd}tj||||jd}||fS)Nscale zero_pointr)rzerosrr-r.quantize_per_tensorr0r1) r2rOr>hch_scaleh_zpc_scalec_zpr7r7r8r<s    zLSTMCell.initialize_hiddencCdS)NQuantizableLSTMCellr7r2r7r7r8 _get_namezLSTMCell._get_namecCs(|du|duks J|jd}|jd}||||du|d}|sJtj||j_|dur4tj||j_tj||j_|durHtj||j_|St||g||g|j|jgD]9\} } } t| j ddd| D] \} } tj| | _qi| durt| j ddd| D] \}} tj|| _qqX|S)zUses the weights and biases to create a new LSTM cell. Args: wi, wh: Weights for the input and hidden layers bi, bh: Biases for the input and hidden layers Nr;)rr r rrr)dim) r=rr Parameterrweightr rrAr@rC)clswiwhbibhrrrrwbr!w_chunkrIb_chunkr7r7r8 from_paramss2   &  zLSTMCell.from_paramscCst||jks Jt|dsJd|j|j|j|j|j|d}|j|_|j|j _|j|j _|rH|j D]}|j|_q5|j D]}|j|_qA|S)Nqconfigz$The float module must have 'qconfig'r) type _FLOAT_MODULEhasattrrl weight_ih weight_hhbias_ihbias_hhrmrrrC)rcotheruse_precomputed_fake_quantrobservedr4r7r7r8 from_floats$    zLSTMCell.from_floatTNNN)F)NNF)FF)__name__ __module__ __qualname____doc__rrrro __constants__intboolrrrtuplerNr<r^ classmethodrlrx __classcell__r7r7r5r8rsN ;  ,   $c sleZdZdZ   ddddededed dffd d Zdd ed ee eeffddZ e ddZ Z S)_LSTMSingleLayerzA single one-directional LSTM layer. The difference between a layer and a cell is that the layer can process a sequence, while the cell only expects an instantaneous value. TNFrrr r r cs2||d}tt||f||d||_dSNr )r r)rrrr)r2rr r r rrr3r5r7r8rs  z_LSTMSingleLayer.__init__r9r:cCsNg}|jd}t|D]}||||}||dq t|d}||fS)Nr)r=rangerappendrstack)r2r9r:resultseq_leni result_tensorr7r7r8rNs   z_LSTMSingleLayer.forwardcOs2tj|i|}||j|j|j|jd}||_|S)Nr)rrlrrr rr)rcargskwargsrlayerr7r7r8rls z_LSTMSingleLayer.from_paramsryrz)r{r|r}r~rrrrrrrNrrlrr7r7r5r8rs&    rcszeZdZdZ     ddddededed ed ed df fd d Zddedee eeffddZ e dddZ Z S) _LSTMLayerz#A single bi-directional LSTM layer.TFNrrr r batch_first bidirectionalr c sb||d} t||_||_t||f||d| |_|jr/t||f||d| |_dSdSr)rrrrrlayer_fwlayer_bw) r2rr r rrr rrr3r5r7r8rs,  z_LSTMLayer.__init__r9r:cCs|jr |dd}|durd\}}n|\}}d}|jrE|dur"d}n|d}|d}|dur1d}n|d}|d}|durE|durE||f}|durP|durPd}n tj|tj|f}|||\} }t|dr|jr|d} | | |\} }| d} t | | g| d} |dur|durd} d}n=|durtj|\} }n0|durtj|\} }n#t |d|dgd} t |d|dgd}n | } tj|\} }|jr| dd| | |ffS)Nrr;)NNr)r transposerrjit_unwrap_optionalrrpfliprcatr`r transpose_)r2r9r:hx_fwcx_fw hidden_bwhx_bwcx_bw hidden_fw result_fw x_reversed result_bwrrUrVr7r7r8rN*sT      z_LSTMLayer.forwardrcKsPt|ds |dus J|d|j}|d|j}|d|j}|d|j}|d|j} |dd } |||||| | d } t|d|| _t|d |} t|d |} t|d |d}t|d|d}t j | | ||| d | _ |jrt|d |d} t|d |d} t|d |dd}t|d|dd}t j | | ||| d | _ | S)z There is no FP equivalent of this class. This function is here just to mimic the behavior of the `prepare` within the `torch.ao.quantization` flow. rmNrrr rrrFr weight_ih_l weight_hh_l bias_ih_l bias_hh_l_reverse) rpgetrrr rrgetattrrmrrlrr)rcru layer_idxrmrrrr rrrrrdrerfrgr7r7r8rx`s@   z_LSTMLayer.from_float)TFFNNrz)rN)r{r|r}r~rrrrrrrNrrxrr7r7r5r8r s2   6rcseZdZdZejjZ       dddded ed ed e d e d e de de ddffddZ dde de ee e ffddZddZedddZeddZZS) raXA quantizable long short-term memory (LSTM). For the description and the argument types, please, refer to :class:`~torch.nn.LSTM` Attributes: layers : instances of the `_LSTMLayer` .. note:: To access the weights and biases, you need to access them per layer. See examples below. Examples:: >>> import torch.ao.nn.quantizable as nnqa >>> rnn = nnqa.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) >>> # To get the weights: >>> # xdoctest: +SKIP >>> print(rnn.layers[0].weight_ih) tensor([[...]]) >>> print(rnn.layers[0].weight_hh) AssertionError: There is no reverse path in the non-bidirectional layer r;TFNrrr num_layersr rdropoutrrr c  s|| dt|_|_|_|_|_t|_|_ d_ t |t j r>d|kr4dkr>ntdt |trBtd|dkrZtd|dkrZtd|d|tjjjfdj d g} | fd d td|Dtj| _dS) Nr Frr;zbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=rrrc3s4|]}tjjjfdjdVqdS)FrN)rrr r.0_r3r2rr7r8 s  z LSTM.__init__..)rrrrrr rfloatrrtraining isinstancenumbersNumberr ValueErrorwarningswarnrextendrrr ModuleListlayers) r2rrrr rrrr rrrr5rr8rsb       z LSTM.__init__r9r:cs|jr |dd}|d}|jrdnd}|durEtj|||jtj|jd d|j r8tj dd|j dfddt |jD}n5tj|}t|dtrx|d|j|||j|d|j|||jfd dt |jD}n|}g}g}t|jD]!\} } | ||| \}\} } |tj| |tj| qt|} t|}| d | jd | jd } |d |jd |jd }|jr|dd}|| |ffS) Nrr;)rr rrPcsg|]}fqSr7r7r)rSr7r8 sz LSTM.forward..cs(g|]}|d|dfqS)r)squeeze)ridx)rGrFr7r8rs)rrsizerrrSrrr squeeze_r>rTrrrrrrrreshape enumeraterrrr=)r2r9r:max_batch_sizenum_directionshxcxhidden_non_opthx_listcx_listrrrUrV hx_tensor cx_tensorr7)rGrFrSr8rNsV            z LSTM.forwardcCr[)NQuantizableLSTMr7r]r7r7r8r^+r_zLSTM._get_namec Cst||jsJt|ds|sJ||j|j|j|j|j|j|j |d}t |d||_ t |jD]}t j|||d|d|j|<q0|jrR|tjjj|dd}|S|tjjj|dd}|S)NrmrF)rrT)inplace)rrorprrrr rrrrrmrrrxrrtrainrr quantization prepare_qatevalprepare)rcrurmrrwrr7r7r8rx.s0  zLSTM.from_floatcCstd)NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)rcrur7r7r8 from_observedKszLSTM.from_observed)r;TFrFNNrz)NF)r{r|r}r~rrrrorrrrrrrrNr^rrxrrr7r7r5r8rsJ    J7 )r~rrtypingrrr__all__rModulerrrrr7r7r7r8s  O*