
    8jg                        S SK Jr  S SKrS SKJr  S SKJr  S SKJr  S SK	J
r  S SKJr  S SKJ
r  S SKJr  S S	KJr  S
r/ SQrSr " S S5      rg)    )annotationsN)Path)List)	Tokenizer)	ByteLevel)BPE)TemplateProcessing)
BpeTrainerzklue/roberta-base)[PAD][UNK][CLS][SEP]i>  c                      \ rS rSr          SS jrSS jrSS jrSS jr\  S           SS jj5       r	\SSS jj5       r
\SS j5       r\SS	 j5       rS
rg)MicroTokenizer   c                L    Xl         X l        X0l        X@l        XPl        X`l        g Nbackendpad_idcls_idsep_id
vocab_sizemode)selfr   r   r   r   r   r   s          `/home/ubuntu/service/kemix-engine/package/face/animasync-face-v3/models/microalbert/tokenizer.py__init__MicroTokenizer.__init__   s"     $	    c                    U R                   $ r   )r   )r   s    r   __len__MicroTokenizer.__len__$   s    r   c                    U R                   S:X  a!  U R                  USSUSS9n[        US   5      $ U R                  R                  U5      n[        UR                  5      S U $ NhfTFadd_special_tokens
truncation
max_lengthreturn_attention_mask	input_ids)r   r   listencodeids)r   textmax_lenoutencs        r   r-   MicroTokenizer.encode'   sk    99,,#'"&+  C K())ll!!$'CGG}Xg&&r   c                "   U R                   S:X  a2  U R                  USSUSS9nUS    Vs/ s H  n[        U5      PM     sn$ U R                  R                  U5      nU Vs/ s H  n[        UR                  5      S U PM     sn$ s  snf s  snf r$   )r   r   r,   encode_batchr.   )r   textsr0   r1   r.   	encodingses          r   batch_encodeMicroTokenizer.batch_encode4   s    99,,#'"&+  C *-[)9:)9#DI)9::LL--e4	/89y!QUUHW%y99 ;9s   B""Bc                   [        U5      nUR                  SSS9  US-  nUR                  5       (       a  U(       d   U R                  U5      $ U(       a   U R                  X5S9$ U R                  X!U5      $ ! [         a  n[        SU 35         S nANFS nAff = f! [         a  n[        SU S35         S nANUS nAff = f)NT)parentsexist_oktokenizer.jsonz.[tokenizer] local tokenizer.json load failed: )add_speaker_tokensz[tokenizer] HF path failed: z; training fresh BPE on seed)r   mkdirexists	_load_bpe	Exceptionprint_load_hf
_train_bpe)clssave_dirtrain_jsonlr0   	prefer_hfr?   existingr8   s           r   buildMicroTokenizer.buildA   s     >td3..??%7L}}X.. V||G|SS ~~kW==  LFqcJKKL
  V4QC7STUUVs/   A8 B 8
BBB
C 'B;;C c           	        SSK Jn  UR                  [        5      nU(       a  UR	                  SSS/05        U" SSSUSS	9n[        US
   5      S:  d   S5       eUR                  b  UR                  c   eU " U[        UR                  5      [        UR                  5      [        UR                  5      [        U5      SS9$ )Nr   )AutoTokenizeradditional_special_tokensz[SELF]z[OTHER]u   안녕하세요TFr&   r+      zHF tokenizer smoke test failedr%   r   )
transformersrO   from_pretrained	_HF_MODELr'   lenpad_token_idcls_token_idintsep_token_id)rG   r0   r?   rO   tokprobes         r   rE   MicroTokenizer._load_hfZ   s    .++I6 ""$?(IAV#WX#"'
 5%&!+M-MM++0@0@0LLLs''(s''(s''(3x
 	
r   c           	        / n[        U5      R                  SS9 nU HR  n[        R                  " U5      nUS    H0  nUS   R	                  5       n	U	(       d  M  UR                  U	5        M2     MT     S S S 5        [        [        SS95      n
[        SS9U
l	        [        5       U
l        [        [        [        [        R                  " 5       S	9nU
R!                  XKS
9  U
R#                  S5      nU
R#                  S5      nU
R#                  S5      n[%        SSU4SU4/S9U
l        U
R)                  US9  US-  nU
R+                  [-        U5      5        U " U
UUUU
R/                  5       SS9$ ! , (       d  f       GN= f)Nzutf-8)encodingturnsr/   r   )	unk_tokenF)add_prefix_space)r   special_tokensinitial_alphabet)trainerr   r   r   z[CLS] $A [SEP])singlerb   )r)   r>   bper   )r   openjsonloadsstripappendr   r   ByteLevelPreTokpre_tokenizerByteLevelDecoderdecoderr
   _BPE_VOCAB_SIZE_SPECIALalphabettrain_from_iteratortoken_to_idr	   post_processorenable_truncationsavestrget_vocab_size)rG   
jsonl_pathrH   r0   r6   flinerowttxtrZ   rd   r   r   r   out_paths                   r   rF   MicroTokenizer._train_bpeu   sm   *""G"4jj&WAF)//+CsS) &  5 g./+UC&(&#,557

 	7)))/#$f-/@A
 	1..X))+
 	
5 54s   <E/E//
E>c           	         [         R                  " [        U5      5      nU " UUR                  S5      UR                  S5      UR                  S5      UR	                  5       SS9$ )Nr   r   r   rf   r   )r   	from_filerx   rt   ry   )rG   pathrZ   s      r   rB   MicroTokenizer._load_bpe   sY    !!#d),??7+??7+??7+))+
 	
r   )r   r   r   r   r   r   N)
r   rX   r   rX   r   rX   r   rX   r   rx   )returnrX   )r/   rx   r0   rX   r   z	List[int])r6   z	List[str]r0   rX   r   zList[List[int]])TF)rH   r   rI   r   r0   rX   rJ   boolr?   r   r   'MicroTokenizer')F)r0   rX   r?   r   r   r   )rz   r   rH   r   r0   rX   r   r   )r   r   r   r   )__name__
__module____qualname____firstlineno__r   r!   r-   r9   classmethodrL   rE   rF   rB   __static_attributes__ r   r   r   r      s      	
    ':  #(>> > 	>
 > !> 
> >0 
 
4 #
 #
J 	
 	
r   r   )
__future__r   rh   pathlibr   typingr   
tokenizersr   tokenizers.decodersr   rn   tokenizers.modelsr   tokenizers.pre_tokenizersrl   tokenizers.processorsr	   tokenizers.trainersr
   rT   rq   rp   r   r   r   r   <module>r      s>    "      = ! B 4 *	/R
 R
r   