
     hB1                        d dl mZ d dlZd dlmZmZmZmZ d dlmZm	Z	m
Z
 d dlmZ  e            Zd Zd(d
Zd)dZd*dZd+dZd,dZd-d.dZd/dZd,dZd0d1d Zd,d!Zd"Zd,d#Zd2d3d'ZdS )4    )annotationsN)AnyDictListTuple)datetimetimezonedate)
get_clientc                 b    t          j        t          j                                                  S N)r   nowr	   utc	isoformat     '/var/www/html/bot/app/sync2/pipeline.py_now_isor      s     <%%//111r   sstrreturnc                V    t          j        dd| pd                                          S )N\s+ )resubupperr   s    r   
_norm_namer      s&    6&"qwB((..000r   setc                ^    t          d t          j        d| pd          D                       S )Nc                    g | ]}||S r   r   ).0ts     r   
<listcomp>z _dept_tokens.<locals>.<listcomp>   s    BBBaBBBBr   u   [\/\s>\-·]r   )r    r   splitr   s    r   _dept_tokensr'      s/    BB28NAG<<BBBCCCr   abfloatc                t    | s|sdS t          | |z            t          dt          | |z                      z  S )N           )lenmax)r(   r)   s     r   _jaccardr0      s?     Q sq1u::As1q5zz****r   c                    t          j        dd| pd          t          j        dd|pd          rsdS k    rdS g d}t          fd|D                       rdndS )Nr   r   r,         ?)u	   사무관u	   서기관u	   주무관u   국장u   과장u   실장c              3  (   K   | ]}|v o|v V  d S r   r   )r#   kABs     r   	<genexpr>z"_pos_similarity.<locals>.<genexpr>"   s0      88aqAv(!q&888888r   g333333?)r   r   any)r(   r)   keysr5   r6   s      @@r   _pos_similarityr:      s    
vr17##A
vr17##A A sAvvsPPPD88888488888A33cAr   batch_idc                    t                               d                              | dd                                           d S )Ningest_batchesRUNNING)r;   status)sbtableinsertexecute)r;   s    r   start_batchrD   %   s@    HH%%	' '  wyyyyyr   	COMPLETEDsource_summaryDict[str, int] | Nonec                    t                               d                              t                      ||pi d                              d|                                            d S )Nr=   )finished_atr?   rF   r;   )r@   rA   updater   eqrC   )r;   r?   rF   s      r   finish_batchrL   *   s\    HH%%zzV(.B' '   
r*h					r   rowsList[Dict[str, Any]]c                   g }|D ]}|                     | |d         |                    d          pd                                |                    d          |                    d          |                    d          |                    d          |                    d          d	           |rAt                              d
                              |                                           d S d S )Nsourcenamer   
departmentpositionphonetaskraw)r;   rP   rQ   rR   rS   rT   rU   rV   staging_gov_staff)appendgetstripr@   rA   rB   rC   )r;   rM   payloadrs       r   ingest_stagingr]   1   s    G 
 
 kUU6]](b//11%%--j))UU7^^EE&MM55<<	
 	
 		 		 		 		  @
$%%,,W55==?????@ @r   c                T   t                               d                              d                              d|                                           j        pg }t                               d                              d                                          j        pg }t                               d                              d                                          j        pg }i }|D ]1}|                    |d         g                               |           2g }|D ]}t          |d                   }|	                    d	          pd

                                t          |	                    d          pd
          }	|	                    d          pd
}
|D ]A}t          |d                   |k    rd}r8t          fd|	                    |d         g           D                       rd}nd}d}|	                    |d         g           D ]s}t          |t          |	t          |	                    d          pd
                              }t          |t          |
|	                    d                              }tt!          t#          d|dz  |dz  z             d          }|dk    r'|                    | |d         |d         |i d           C|rAt                               d                              |                                           d S d S )NrW   *r;   person_registryzperson_id,canonical_namegov_staff_current	person_idrQ   rT   r   rR   rS   canonical_namer,   c              3  N   K   | ]}|                     d           pdk    V   dS )rT   r   NrY   )r#   xs_phones     r   r7   z#build_candidates.<locals>.<genexpr>Y   s7      vvqAEE'NN,@b Avvvvvvr   r2   g?g      ?g?   
staging_id)r;   ri   candidate_person_idscorefeaturesmatch_candidates)r@   rA   selectrK   rC   data
setdefaultrX   r   rY   rZ   r'   r8   r/   r0   r:   roundminrB   )r;   stpersonscurrentcurrent_by_personr\   candsr   s_name_norms_depts_posprk   	best_deptbest_posrf   rg   s                   @r   build_candidatesr~   B   s*   	%	&	&	-	-c	2	2	5	5j(	K	K	S	S	U	U	Z	`^`Bhh())001KLLTTVV[a_aGhh*++22377??AAFL"G/1 C C$$Q{^R88??BBBBE   6++55>>'R..00quu\228b99%%
##)r 	 	A!,-..+==E 	Q3vvvvL]LaLabcdobprtLuLuvvvvv 	Q  	*..q~rBB Y YA #IxQUUS_M`M`Mfdf@g@g/h/h i iI #HoeQUU:EVEV.W.W X XHHc#	CHsN'KLLaPPs{{ ("#L/+,[>" "    '	6  =
#$$++E22::<<<<<= =r   333333?皙?himarginc           	        t                               d                              d                              d|                               d                              dd                                          j        pg }i }|D ]1}|                    |d         g                               |           2t                               d                              d                              d|                                           j        pg }d	 |D             }g }|	                                D ]\  }	}
|
d
         }t          |d                   }t          |
          dk    rt          |
d         d                   nd}||k    r+||z
  |k    r"|                    | |	|d         d|d           |                    | |	d d|d           t          |                                          }|D ]!}||vr|                    | |d ddd           "|rF|D ]E}t                               d                              |d                                           Dd S d S )Nrm   r_   r;   ri   rk   T)descrW   c                    g | ]
}|d          S ri   r   r#   r\   s     r   r%   zauto_decide.<locals>.<listcomp>}   s    333a!L/333r   r   r-   r,   rj   AUTO)r;   ri   rb   decisionrk   PENDINGr2   match_decisionszbatch_id,staging_idon_conflict)r@   rA   rn   rK   orderrC   ro   rp   rX   itemsr*   r.   r    r9   upsert)r;   r   r   rM   
by_stagingr\   st_rows
all_st_ids	decisionsri   rw   top	top_scoresecondstaged_with_candssidds                    r   auto_decider   q   s   88&''..s33	J	!	!%%"5"5eeG$e6O6O	4 	
 )+J = =aor2299!<<<< hh*++22<@@CCJPXYYaacchnlnG337333JI (--//  
EAh#g,''	-0ZZ!^^uQx()))??	F 2v==$( !67""      $(!%"      JOO--..  '''$!!"      _ 	_ 	_AHH&''..q>S.TT\\^^^^_ _	_ 	_r   c           
     F   t                               d                              d                              d|                                           j        pg }g }|D ]~}t                               d                              d|d         i                                          j        }|d         d         }|                    | |d	         |d
ddd           |rAt                               d                              |                                           d S d S )NrW   r_   r;   r`   rc   rQ   r   rb   ri   r   r2   	bootstrap)r;   ri   rb   r   rk   	rationaler   )r@   rA   rn   rK   rC   ro   rB   rX   )r;   rs   decsr   r\   pids         r   seed_batch_as_newr      s$   	%	&	&	-	-c	2	2	5	5j(	K	K	S	S	U	U	Z	`^`BD 
 
HH&''..0@!F)/LMMUUWW\d; L/$
 
 	 	 	 	  ;
"##**40088:::::; ;r   Tc                h   t                               d                              d                              d|                                           j        pg }d |D             }t                               d                              d                              d|                                           j        pg }t                               d                              d                                          j        pg }d |D             }d0d}g }g }t                      }	|D ]r}
|
d         dvr|                    |
d                   }|s,|
                    d          }|sZt                               d                              d|d         i                                          j        }|d         d         }t                               d          	                    dt                      i                              d|                                           ||d         f}|	                    |           |                    |          }||d         |                    d          |                    d          |                    d          |                    d          |                    d          d| t                      d
}g }|dg}nY|                    d          pd|                    d          pdk    r|                    d           |                    d          pd|                    d          pdk    r|                    d            |                    d          pd|                    d          pdk    r|                    d!           |                    d          pd|                    d          pdk    r|                    d"           |                    d          pd|                    d          pdk    r|                    d#           |                    | ||d         |rd$                    |          nd% ||           ||          d&           |                    |           t|                                D ]w\  \  }}}|                    d'          rZ||f|	vrT|                    | ||d( ||          d)d)d)d)d)d)d*d&           |                    i |d+t                      d,           x|r?t                               d-                              |                                           |D ]C}t                               d                              |d./                                           Dd S )1NrW   r_   r;   c                     i | ]}|d          |S r   r   r   s     r   
<dictcomp>z#apply_decisions.<locals>.<dictcomp>   s    444q,444r   r   ra   c                0    i | ]}|d          |d         f|S )rb   rP   r   r   s     r   r   z#apply_decisions.<locals>.<dictcomp>   s'    CCCQ;8-qCCCr   rf   dict | Noner   dictc                   | pi } |                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pd|                      d          pddS )	NrP   -rR   rS   rQ   rT   rU   rP   rR   rS   rQ   rT   rU   re   )rf   s    r   	fill_fullz"apply_decisions.<locals>.fill_full   s    G%%//4%%--4%%
++4%%--4%%..4%%--4
 
 	
r   r   )r   MANUALri   rb   r`   rc   rQ   r   last_seen_atrP   rR   rS   rT   rU   T)
rb   rP   rQ   rR   rS   rT   rU   	is_activelast_seen_batch
updated_atNEWr   MOVECHANGE_POSITIONCHANGE_PHONECHANGE_TASKCHANGE_NAME,	NO_CHANGE)r;   rb   rP   
event_typeprevnextr   DEPARTEDr   r   F)r   r   gov_staff_eventszperson_id,sourcer   )rf   r   r   r   )r@   rA   rn   rK   rC   ro   r    rY   rB   rJ   r   addrX   joinr   r   )r;   r   st_by_idr   curr_allcurr_keyr   eventsupsertsseenr   r   r   r\   keyr   next_rowkindssrcrows                       r   apply_decisionsr      s   hh*++22377:::xPPXXZZ_eceG44G444H88%&&--c2255j(KKSSUUZ`^`Dxx+,,33C88@@BBGM2HCC(CCCH

 

 

 

 FG55D :! :!Z= 222LL<)) 	 eeK   	$*++224Dai3PQQYY[[`AA${#C 	"##**NHJJ+GHHKKKY\]]eegggAhK ||C  kEE&MM%%--j))UU7^^EE&MM'"**
 
 <GEE&&,"(,,|2L2L2RPRSSV$$$$$*Z0H0H0NBOO.///!!'RX\\'-B-B-HbII^,,,  &BHLL,@,@,FBGG]+++  &BHLL,@,@,FBGG]+++ k-2C#((5///IdOOIh''
 
 	 	 	 	x     %NN,, S S
cD88K   	Sc3Zt%;%;MM$ (!	$!#Cc 
 
 
 
 
 NNQdQhjjQQQRRR >
#$$++F33;;===  \ \
$%%,,S>P,QQYY[[[[\ \r     limitintc           
        	 t                               d                              d                              dd                              |                                           j        }|rd |D             S n# t          $ r Y nw xY w	 t                               d                              d                              |                                           j        }|rqg }|D ]j}|                    d|	                    d          |	                    d	          |	                    d
          |	                    d          |d           k|S n# t          $ r Y nw xY w	 t                               d                              d                              |                                           j        }|rqg }|D ]j}|                    d|	                    d          |	                    d	          |	                    d
          |	                    d          |d           k|S n# t          $ r Y nw xY wg S )N	gov_staffzname,department,position,phonerP   MOTIEc                "    g | ]}d di|d|iS )rP   r   rV   r   r   s     r   r%   z,load_motie_from_existing.<locals>.<listcomp>0  s,    EEE1Xw6!6UA66EEEr   motie_org_peopler_   rQ   rR   rS   rT   )rP   rQ   rR   rS   rT   rV   	motie_org)
r@   rA   rn   rK   r   rC   ro   	ExceptionrX   rY   )r   ro   rM   r\   s       r   load_motie_from_existingr   +  s~   xx$$++,LMMPPQY[bcciijoppxxzz 	FEEEEEE	F   xx*++22377==eDDLLNNS 	D  %EE&MM"#%%"5"5 !j 1 1UU7^^      K	    xx$$++C0066u==EEGGL 	D  %EE&MM"#%%"5"5 !j 1 1UU7^^      K	     Is8   A8A< <
B	B	C	E 
E%$E%)C	H4 4
I I)r   r   r   r   )r   r   r   r    )r(   r    r)   r    r   r*   )r(   r   r)   r   r   r*   )r;   r   )rE   N)r;   r   rF   rG   )r;   r   rM   rN   )r   r   )r;   r   r   r*   r   r*   )r   )r   r   r   rN   )
__future__r   r   typingr   r   r   r   r   r	   r
   app.services.supabase_servicer   r@   r   r   r'   r0   r:   rD   rL   r]   r~   r   r   NAME_CHANGED_AS_NEWr   r   r   r   r   <module>r      s   " " " " " " 				 ) ) ) ) ) ) ) ) ) ) ) ) - - - - - - - - - - 4 4 4 4 4 4Z\\2 2 21 1 1 1D D D D+ + + +
B B B B   
* * * * *@ @ @ @",= ,= ,= ,=^6_ 6_ 6_ 6_ 6_r; ; ; ;$  j\ j\ j\ j\Z- - - - - - -r   