
     hh#                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZm	Z	 d dl
m
Z
mZ d dlZd dlmZ d dlmZ  e j        dd                                          Z ej        ed	            ej        d
          ZdZedz   ZdZddiZ ej        d          Zdedz  defdZdedefdZdedededefdZ dedededededefdZ!dededefdZ"d7d"ej#        d#edej$        fd$Z%d"ej#        dee         fd%Z&d"ej#        d&ede	ee         eef         fd'Z'de(fd(Z)d)e(d*e(d+e(d,e*de	e+ef         f
d-Z,deeef         fd.Z-d/edeeef         fd0Z.d/efd1Z/d8d4e*d+e(d,e*defd5Z0e1d6k    r e2 e0                       dS dS )9    N)ListDictTuple)datetimetimezone)BeautifulSoup)
get_client	LOG_LEVELINFOz[%(levelname)s] %(message)s)levelformatmoef_org_pipelinezhttps://www.moef.go.krz;/mi/orgnzt/org.do?bbsId=MOSFBBS_000000000097&menuNo=9040100za[href*="orgId="]z
User-Agentz*GovBot/1.0 (+https://work.jjickjjicks.com)z\d{2,4}-\d{3,4}-\d{4}sreturnc                 |    d                     | pd                    dd                                                    S )N      )joinreplacesplitr   s    2/var/www/html/bot/app/crawler/moef_org_pipeline.py_cleanr      s3    172C2CHS2Q2Q2W2W2Y2Y)Z)Z"Z    c                      t          |           S )N)r   r   s    r   _normr      s    *r   nameposition
departmentc                     t          |            dt          |           dt          |           }t          j        |                                                                          S N|r   hashlibsha256encode	hexdigest)r   r   r    bases       r   	_key_hashr*      sU    DkkAAE(OOAAeJ.?.?AAD>$++--((22444r   taskphonec                    t          |            dt          |           dt          |           dt          |           dt          |           	}t          j        |                                                                          S r"   r$   )r   r   r    r+   r,   r)   s         r   	_row_hashr.      su    Dkk^^E(OO^^eJ.?.?^^%++^^PUV[P\P\^^D>$++--((22444r   name_rawposition_rawc           	      @   t          |           }t          |          }|r|s|S |}t          j        d|          d                                         }d ||hD             }|D ]%}|d|z   fD ]O}|                    |          r8t          |d t          |                                                  d          }Pt          j        dt          j        |           dd|                              d          }t          j        d	t          j        |           d
d|                              d          }|                    |          r8t          |d t          |                                                  d          }'t          |          }|r|n|S )Nu   [ \t(\[/·\-]r   c                     g | ]}||S  r3   ).0cs     r   
<listcomp>z,strip_position_from_name.<locals>.<listcomp>#   s    ,,,!,!,,,r   r   u    -·/–—z\s*\(\s*z	\s*\)\s*$r   z\s*z\s*\([^)]+\)\s*$)	r   rer   stripendswithlenrstripsubescape)	r/   r0   nporiginalr)   
candidatescandsufs	            r   strip_position_from_namerD      s   xAf\22!AaxH8$a((+1133D,,aY,,,J O O#*% 	Q 	QCzz#PF1[Cy[>$:$:$A$A-$P$PF9ry9992qAAHHWWF;")D//;;;RCCJJ=YY::dN,SYYJ,!8!8!?!?!N!NQq		A11xr         ffffff?sessurlc           	      D   d }t          |          D ]}	 |                     |t          |          }|                                 |c S # t          $ rI}|}|d|z  z  }	t
                              d|dz   ||	|           t          j        |	           Y d }~d }~ww xY w|)N)headerstimeout   zGET retry %d/%d in %.1fs: %s   )	rangegetHEADERSraise_for_status	Exceptionloggerwarningtimesleep)
rH   rI   rL   triesbackofflastirerW   s
             r   
_retry_getr^   -   s    D5\\ f f	fgw??AASASAUAUAU^_W_W_W_ 	f 	f 	fDgq!tneNN91Q3ucRRRTXT^_dTeTeTeTeTeTeTeTe	f Js   2A


B?BBc                 (   t          | t                    }t          |j        d          }t	                      g }}|                    t                    D ]}|                    dd          pd                                }|rd|vr3|	                    d          r|n+|	                    d          r
t          |z   nt          dz   |z   }||v r{|                    |           |                    ||                    dd          d           |S )	Nhtml.parserhrefr   zorgId=http/title)linkrd   )r^   LIST_URLr   textsetselect	SEL_LINKSrP   r8   
startswithBASEaddappend)rH   r\   soupseenoutara   re   s           r   fetch_org_linksrs   7   s   4""A//Dr#D[[## ? ?fR  &B--//5xt++Xv..pttDOOTWDXDX5oTD[[^beh^hko^o4<<

D155+<+<==>>>>Jr   re   c           	         t          | |          }|j        }t          |d          }g }|                    d          D ]}|                    d          r|                    d          }t          |          dk     rBt          |d                             d                    }t          |d	                             d                    }	t          |d
                             d                    }
t          |d                             d                    }t          |d                             d                    }|d         	                    d          }|rC|
                    d          r.|d                             dd                                          }t          |	|
          }|
}|s
|s|s|s|s|                    |||||d           |||fS )Nr`   ztable trthtd   r   T)r8   rN   rM   rF      za[href^="tel:"]ra   ztel:r   )r    r   r   r,   r+   )r^   rg   r   ri   findfind_allr:   r   get_text
select_onehas_attrr   r8   rD   rn   )rH   re   r\   htmlro   rowstrtdsr    r/   r0   r,   responsibilitytel_ar   r   s                   r   parse_org_detailr   D   s   4Aqv}--DDkk*%% w w774=="(kk$s88A::xAd ; ;<<
Ad ; ;<<Ad ; ;<<Ad ; ;<<Ad ; ;<<A!!"344]U^^F++]U6]5J5J6RT5U5U5[5[5]5]U',??TTJT.TETH*ThX]ftuuvvvvtr   c                     	 |                      d                              dd                                          }|j        pdS # t          $ r Y dS w xY w)Nmoef_org_curkey_hashexact)countr   )tableri   executer   rS   )sbress     r   
_count_curr   Y   se    hh~&&--j-HHPPRRy~A   qqs   AA 
AA	stg_count	cur_countmin_abs	min_ratioc                    | |k     r
dd|  d| fS |dk    r4| t          ||z            k     rdd|  dt          ||z             d| dfS dS )	NFztoo_few_rows: z < r   zratio_drop: z (cur=))Tok)int)r   r   r   r   s       r   	_validater   `   s    75*R9*R*R*R*R#RR{{y3y':#;#;;;_Y__3y7J3K3K__S\_____:r   c                     |                      d                              d                              d                                          j        pg }d |D             S )Nr   *P  c                      i | ]}|d          |S r   r3   r4   r\   s     r   
<dictcomp>z%_load_current_map.<locals>.<dictcomp>h       ---!Qz]A---r   )r   ri   limitr   data)r   r   s     r   _load_current_mapr   f   sX    HH^$$++C0066u==EEGGLSQSD--t----r   run_idc                     |                      d                              d                              d|                              d                                          j        pg }d |D             S )Nmoef_org_stgr   r   r   c                      i | ]}|d          |S r   r3   r   s     r   r   z!_load_stg_map.<locals>.<dictcomp>l   r   r   )r   ri   eqr   r   r   )r   r   r   s      r   _load_stg_mapr   j   sh    HH^$$++C0033HfEEKKERRZZ\\ahfhD--t----r   c                 ~   t          |           }t          | |          }g g }}t          j        t          j                                                  }|                                D ]}||vr|                    |           |	                                D ]\  }}||vs||         d         |d         k    r^|                    ||d         |d         |d         |
                    d          |
                    d          |d         |d dd
           |re|                     d	                              |d
d                              d|                              dd                                           |r<|                     d	                              |                                           d S d S )Nrow_hashr    r   r   r+   r,   T)
r   r    r   r   r+   r,   r   
valid_fromvalid_to
is_currentmoef_org_histF)r   r   r   r   )r   r   r   nowr   utc	isoformatkeysrn   itemsrP   r   updatein_r   r   insert)	r   r   cur_mapstg_mapto_closeto_addnow_isokvs	            r   _apply_scd2r   n   s   ##GB''G2fHl8<((2244G\\^^ 0 0GX__Q///  1W'!*Z"8AjM"I"IMMQ|_ai]^_i]jfgjMdbf    
  P
!!((gU)S)STTXXYcemnnqqr~  AE  	F  	F  	N  	N  	P  	P  	P ;
!!((0088:::::; ;r   皙?   	sleep_secc                 `   t                      }t          t          j                              }|                    d                              |ddd                                           d}d}	 t          j                    5 }t          |t                    }|                    d                              |ddt          |j        d                                           t          |          }	t          |	d	          D ]\\  }
}t          ||d
                   \  }}}|                    d                              |d|
||d                                           |rg }|D ]}t!          |d         |d         |d                   }t#          |d         |d         |d         |                    dd          |                    dd                    }|                    d|i|||d           |                    d                              |                                           |t)          |          z  }|d	z  }t+          j        |            ^	 d d d            n# 1 swxY w Y   |                    d                              ||dd| d                                           t/          |          }t1          ||||          \  }}|s|                    d                              dt5          j        t8          j                                                  |||d                              d|                                           |                    d                              d|d                              d|                                           t@          !                    d|           |d|dS tE          ||           |                    d                              d d!i                              d|                                           |                    d                              d!t5          j        t8          j                                                  ||d"                              d|                                           t@          #                    d#|||           |d!||d$S # tH          $ r}|                    d                              d%t5          j        t8          j                                                  ||t          |          d                              d|                                           t@          %                    d&|           |d%t          |          d'cY d }~S d }~ww xY w)(Ncrawler_runmoef_orgrunning)idtargetstatusr   moef_org_rawlist)r   kindpagerI   r~   rN   re   detailr   r   r    r+   r   r,   r   )r   r   r   moef_org_snapshot	collectedzpages=)r   r   r   note)r   r   failed)r   finished_atpagesr   fail_reasonr   )r   r   z [moef_org] validation failed: %s)r   r   reasonr   passed)r   r   r   r   z3[moef_org] run passed: run_id=%s, rows=%d, pages=%d)r   r   r   r   abortedz[moef_org] run aborted: %s)r   r   error)&r	   struuiduuid4r   r   r   requestsSessionr^   rf   upsertrg   rs   	enumerater   r*   r.   rP   rn   r:   rV   rW   r   r   r   r   r   r   r   r   r   rT   r   r   inforS   	exception)r   r   r   r   r   r   r   rH   list_rlinksr[   itemr   rI   r~   payloadr\   r   hr   r   r   r]   s                          r   run_oncer      s   	BFHH]""&:PY#Z#Z[[cceeeE1y)G 	J4h//FHH^$$++ &!HV\Va- -  wyyy#D))E$UA.. J J4"24f"F"Fc4((//$h#W[1 1  7999 G G! ^ ^%ai:,PP%ai:,QRQVQVW]^`QaQacdchchipqsctctuu&'\A'\1Z['\'\'\]]]]HH^,,33G<<DDFFFSYY&	
DJy4I4I4I4IJ	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J 	J, 	$%%,,	\g  rBz  rB  rB  .C  .C  	D  	D  	L  	L  	N  	N  	NrNN	y)WPYZZZ
F 	KHH]##**XHLYaYeLfLfLpLpLrLr  |A  IR  ag  ,h  ,h  i  i  l  l  mq  sy  z  z  B  B  D  D  DHH())00861R1RSSVVW_aghhpprrrLL;VDDD$x6JJJB
$%%,,hx-@AADDXvVV^^```
&&U]UaHbHbHlHlHnHnw|  EN  (O  (O  	P  	P  	S  	S  TX  Z`  	a  	a  	i  	i  	k  	k  	kI6S\^cddd 8YQVWWW G G G
&&V^VbIcIcImImIoIox}  FO  ^a  bc  ^d  ^d  (e  (e  	f  	f  	i  	i  jn  pv  	w  	w  	  	  	A  	A  	A5q999 9s1vvFFFFFFFFGsL   2S GI/#S /I33S 6I37ES DS 
V-&B<V("V-(V-__main__)rE   rF   rG   )r   r   rG   )3osr7   rV   r   r%   loggingtypingr   r   r   r   r   r   bs4r   app.services.supabase_servicer	   getenvupperr
   basicConfig	getLoggerrT   rl   rf   rj   rQ   compilePHONE_REr   r   r   r*   r.   rD   r   Responser^   rs   r   r   r   floatboolr   r   r   r   r   __name__printr3   r   r   <module>r      s   + + + + + + + + + + + + + + + + + + + + + + + + $ $ $ $ $ $ $ $ $ $ ' ' ' ' ' ' ' '        4 4 4 4 4 4BIk6**0022	  ),I J J J J		.	/	/PP	E
F2:.// ZcDj ZS Z Z Z Z *S *S * * * *5C 53 5C 5C 5 5 5 55C 53 5C 5s 53 5SV 5 5 5 5 s  #  #         X% C V^Vg    (* tDz    8+ 3 5dSRUAU;V    *c     s s e PUVZ[^V^P_    .T#t)_ . . . ..S .T#t)_ . . . .;3 ; ; ; ;&/G /G /Gc /GE /GTX /G /G /G /Gb z	E((** r   