
    8ja                       S SK Jr  S SKrS SKJr  S SKJr  SSKJr   " S S\R                  5      r
 " S	 S
\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      rg)    )annotationsN)nn)
functional   )MicroAlbertConfigc                  6   ^  \ rS rSrSU 4S jjrSS jrSrU =r$ )MicroAlbertEmbeddings
   c                t  > [         TU ]  5         [        R                  " UR                  UR
                  UR                  S9U l        [        R                  " UR                  UR
                  5      U l	        [        R                  " UR
                  UR                  SS9U l        [        R                  " UR                  UR                  S9U l        [        R                   " UR"                  5      U l        U R%                  S[&        R(                  " UR                  5      R+                  S5      SS9  g )N)padding_idxF)biasepsposition_idsr   )
persistent)super__init__r   	Embedding
vocab_sizeembedding_sizepad_token_id	token_embmax_seq_lenpos_embLinearhidden_sizeemb_proj	LayerNormlayer_norm_eps
layer_normDropoutdropoutregister_buffertorcharange	unsqueezeselfcfg	__class__s     \/home/ubuntu/service/kemix-engine/package/face/animasync-face-v3/models/microalbert/model.pyr   MicroAlbertEmbeddings.__init__   s    NNC..C<L<L
 ||COOS5G5GH		#"4"4cooER,,sC<N<NOzz#++.LL)33A6 	 	
    c                    UR                  S5      nU R                  S S 2S U24   nU R                  U5      U R                  U5      -   nU R	                  U5      nU R                  U5      nU R                  U5      $ )Nr   )sizer   r   r   r   r    r"   )r(   	input_idsLposxs        r+   forwardMicroAlbertEmbeddings.forward   sn    NN12A2&NN9%S(99MM!OOA||Ar-   )r"   r   r    r   r   r)   r   )r0   torch.Tensorreturnr7   __name__
__module____qualname____firstlineno__r   r4   __static_attributes____classcell__r*   s   @r+   r	   r	   
   s    
 r-   r	   c                  :   ^  \ rS rSrSU 4S jjrSSS jjrSrU =r$ )MicroAlbertAttention#   c                  > [         TU ]  5         UR                  UR                  -  S:X  d   eUR                  U l        UR                  UR                  -  U l        UR                  U l        [
        R                  " UR                  SUR                  -  5      U l        [
        R                  " UR                  UR                  5      U l        UR                  U l	        g )Nr      )
r   r   r   	num_headshead_dimr   r   qkvout_projattention_dropoutr'   s     r+   r   MicroAlbertAttention.__init__$   s    .!3333==8??99S__a#//.AB		#//3??C!$!6!6r-   c                   UR                   u  pEnU R                  U5      nUR                  U R                  SS9u  pn
UR	                  XEU R
                  U R                  5      R                  SS5      nU	R	                  XEU R
                  U R                  5      R                  SS5      nU
R	                  XEU R
                  U R                  5      R                  SS5      nUR                  [        R                  [        R                  [        R                  [        R                  4;   d   eUR                  5       S S 2S S S S 24   nU R                  (       a  U R                  OSn[         R"                  " XXUS9nUR                  SS5      R%                  5       R	                  XEU R                  5      nU R'                  U5      nU(       a  UXU
4$ U$ )N)dimr              )	attn_mask	dropout_p)shaperH   splitr   viewrF   rG   	transposedtyper$   boollongint32int64trainingrJ   Fscaled_dot_product_attention
contiguousrI   )r(   hattention_mask
return_qkvBr1   _rH   q_flatk_flatv_flatqkvmask4dropouts                    r+   r4   MicroAlbertAttention.forward.   sn   ''ahhqk!$4+;+;!DKKdnndmm<FFq!LKKdnndmm<FFq!LKKdnndmm<FFq!L##

EJJU[['YYYY##%atQ&67)-t%%C,,Q1QUVmmAq!,,.33A$:J:JKmmC ..
r-   )rJ   rG   r   rF   rI   rH   r6   Fr`   r7   ra   r7   rb   rX   r9   r@   s   @r+   rB   rB   #   s    7 r-   rB   c                  :   ^  \ rS rSrSU 4S jjrSSS jjrSrU =r$ )SharedTransformerBlockA   c                  > [         TU ]  5         [        U5      U l        [        R
                  " UR                  UR                  S9U l        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  5      U l        [        R
                  " UR                  UR                  S9U l        [        R                  " UR                  5      U l        g )Nr   )r   r   rB   attnr   r   r   r   attn_lnr   ffn_sizeffn_upffn_downffn_lnr!   r"   r'   s     r+   r   SharedTransformerBlock.__init__B   s    (-	||COO9K9KLii>		#,,@ll3??8J8JKzz#++.r-   c                h   U(       a  U R                  XSS9u  pEpgOU R                  X5      nU R                  XR                  U5      -   5      nU R                  [        R
                  " U R                  U5      5      5      nU R                  XR                  U5      -   5      nU(       a  UWWW4$ U$ )NTrb   )ru   rv   r"   ry   r]   gelurx   rz   )	r(   r`   ra   rb   attn_outrh   ri   rj   ffns	            r+   r4   SharedTransformerBlock.forwardK   s     $		!	 MHAyy3HLL\\(334mmAFF4;;q>23KKLL--.aA:r-   )ru   rv   r"   ry   rz   rx   r6   ro   rp   r9   r@   s   @r+   rr   rr   A   s    /
 
r-   rr   c                  D   ^  \ rS rSrSU 4S jjrSS jrSS	S jjrSrU =r$ )
MicroAlbertBackboneX   c                   > [         TU ]  5         Xl        [        U5      U l        [        U5      U l        UR                  U l        U R                  U R                  5        g N)
r   r   r)   r	   
embeddingsrr   shared_block
num_layersapply_init_weightsr'   s     r+   r   MicroAlbertBackbone.__init__Y   sI    /4237..

4%%&r-   c                n   U R                   R                  n[        U[        R                  5      (       aa  [        R
                  R                  UR                  SUS9  UR                  b*  [        R
                  R                  UR                  5        g g [        U[        R                  5      (       a}  [        R
                  R                  UR                  SUS9  UR                  bF  [        R                  " 5          UR                  UR                     R                  5         S S S 5        g g [        U[        R                  5      (       aS  [        R
                  R!                  UR                  5        [        R
                  R                  UR                  5        g g ! , (       d  f       g = fNrP   )meanstdr)   initializer_range
isinstancer   r   initnormal_weightr   zeros_r   r   r$   no_gradzero_r   ones_r(   moduler   s      r+   r   !MicroAlbertBackbone._init_weightsa      hh((fbii((GGOOFMMO={{&v{{+ '--GGOOFMMO=!!-]]_MM&"4"45;;= %_ . --GGMM&--(GGNN6;;' . %_   (F&&
F4c                    U R                  U5      nS =n=pg[        U R                  5       HB  nU(       a'  XR                  S-
  :X  a  U R                  XBSS9u  pEpgM1  U R                  XB5      nMD     U(       a  XEXg4$ U$ )Nr   Tr}   )r   ranger   r   )	r(   r0   ra   return_last_qkvr`   last_qlast_klast_vis	            r+   r4   MicroAlbertBackbone.forwardp   s    OOI&#'''t'A1!(;#;,0,=,=a\`,=,a)66%%a8	 (
 f,,r-   )r)   r   r   r   r6   r   z	nn.Moduler8   Nonero   )r0   r7   ra   r7   r   rX   )	r:   r;   r<   r=   r   r   r4   r>   r?   r@   s   @r+   r   r   X   s    '(
 
r-   r   c                  J   ^  \ rS rSrSU 4S jjrSS jrS	S jrS
S jrSrU =r	$ )MicroAlbertForEmotionVAD}   c           	       > [         TU ]  5         Xl        [        U5      U l        [
        R                  " [
        R                  " UR                  UR                  5      [
        R                  " 5       5      U l
        [
        R                  " UR                  UR                  5      U l        [
        R                  " [
        R                  " UR                  UR                  5      [
        R                  " 5       [
        R                  " UR                   5      [
        R                  " UR                  UR"                  5      5      U l        U R'                  U R(                  5        g r   )r   r   r)   r   backboner   
Sequentialr   r   Tanhpoolernum_emotionsemotion_headvad_head_hiddenGELUr!   r"   vad_dimvad_headr   r   r'   s     r+   r   !MicroAlbertForEmotionVAD.__init__~   s    +C0mmBIIcoos$OQSQXQXQZ[IIcoos7G7GHIIcoos':':;GGIJJs{{#IIc))3;;7	
 	

4%%&r-   c                n   U R                   R                  n[        U[        R                  5      (       aa  [        R
                  R                  UR                  SUS9  UR                  b*  [        R
                  R                  UR                  5        g g [        U[        R                  5      (       a}  [        R
                  R                  UR                  SUS9  UR                  bF  [        R                  " 5          UR                  UR                     R                  5         S S S 5        g g [        U[        R                  5      (       aS  [        R
                  R!                  UR                  5        [        R
                  R                  UR                  5        g g ! , (       d  f       g = fr   r   r   s      r+   r   &MicroAlbertForEmotionVAD._init_weights   r   r   c                    U R                  X5      nUS S 2SS S 24   nU R                  U5      nU R                  U5      U R                  U5      US.$ )Nr   )emotion_logitsvadpooled)r   r   r   r   )r(   r0   ra   r`   clsr   s         r+   r4    MicroAlbertForEmotionVAD.forward   sT    MM)41ajS!"//7==(
 	
r-   c                B    [        S U R                  5        5       5      $ )Nc              3  f   #    U  H'  oR                   (       d  M  UR                  5       v   M)     g 7fr   )requires_gradnumel).0ps     r+   	<genexpr>6MicroAlbertForEmotionVAD.num_params.<locals>.<genexpr>   s      K&7??917799&7s   11)sum
parameters)r(   s    r+   
num_params#MicroAlbertForEmotionVAD.num_params   s    Kdoo&7KKKr-   )r   r)   r   r   r   r6   r   )r0   r7   ra   r7   r8   dict)r8   int)
r:   r;   r<   r=   r   r   r4   r   r>   r?   r@   s   @r+   r   r   }   s    '(
L Lr-   r   )
__future__r   r$   r   torch.nnr   r]   configr   Moduler	   rB   rr   r   r    r-   r+   <module>r      sg    "   $ %BII 2299 <RYY ."")) "J*Lryy *Lr-   