
    9h                         d dl Z d dlZd dlmZmZ d dlZd dlmZ d dlm	Z	m
Z
 dZe dZddiZd	ej        d
efdZd	ej        ded
ee         fdZd
efdZedk    r e             dS dS )    N)ListDict)BeautifulSoup)
get_clientloggerzhttps://www.motie.go.krz/kor/25/empSearchz
User-AgentzMozilla/5.0sessreturnc                 l   |                      t          t          d          }|                                 t	          |j        d          }|                    d          }|sdS |                     dd          }t          j        d|          }|r"t          |
                    d                    ndS )	N   headerstimeouthtml.parserza.direction.last   onclick zempSearch\.list\((\d+)\))getLIST_URLHEADERSraise_for_statusr   text
select_oneresearchintgroup)r   rsouplastr   ms         3/var/www/html/bot/app/crawler/motie_org_sync_n8n.py_fetch_last_pager"      s    7B77A//D??-..D qhhy"%%G
	-w77A&3qwwqzz???Q&    pagec           	         t            d| }|                     |t          d          }|                                 t	          |j        d          }g }|                    d          D ]	}|                    d          }t          |          dk     r,|d         	                    d	
          }|d         	                    d	
          }	|d         	                    d	
          }
|d         }d}|
                    d          }|rQ|                    d          r<|                    d                              dd                                          }nE|	                    dd	
          }t          j        d|          }|r|                    d          n|}t          |          dk    r:d                    d |dd         D                                                       }nd}|r*|r(|                    |d                                          }|
dk    r d|v rt#          j        d| d|	            t'          ||	|
||g          s|                    ||	|
||d           |S )Nz?pageIndex=r   r   r   ztable tbody trtd   r   Tstripr      r   za[href^="tel:"]hrefztel: z\d{2,4}-\d{3,4}-\d{4}c              3   D   K   | ]}|                     d d          V  dS )r-   Tr(   N)get_text).0r&   s     r!   	<genexpr>z_parse_page.<locals>.<genexpr>7   s2      MMRBKK4K88MMMMMMr#      u   파견u   기획재정부uG   [motie_org] skip(row): dept=파견, task contains '기획재정부' :: z / )nameposition
departmenttaskphone)r   r   r   r   r   r   selectfind_alllenr/   r   replacer)   r   r   r   joinr   debuganyappend)r   r$   urlr   r   rowstrtdsr3   r4   r5   phone_tdr7   tel_a	last_textr    r6   s                    r!   _parse_pagerG      s   
(
($
(
(Cgr22A//DDkk*++ , ,kk$s88a<<1vT**q6???..V__4_00
 r7##$566 	3UYYv&& 	3IIf%%--fb99??AAEE ))#T)::I	2I>>A"#2AGGAJJJE s88a<<88MM3qt9MMMMMSSUUDDD  	3T 	3<<r**0022D !!&74&?&?LvcgvvltvvwwwD(Je<== 	 $
 
 	 	 	 	 Kr#   c                  f   t                      } t          j                    5 }t          |          }g }t	          d|dz             D ]9}|                    t          ||                     t          j        d           :	 ddd           n# 1 swxY w Y   	 | 	                    d          
                                                    dd                                           n.# t          $ r!}t          j        d|            Y d}~nd}~ww xY wd}d	}t	          dt!          |          |          D ]^}||||z            }	|	s| 	                    d                              |	                                           |t!          |	          z  }_t          j        d
| d|            |S )uQ   MOTIE 조직도 전체 크롤링(n8n 방식) -> motie_org 테이블 전체 갱신r   g?N	motie_orgr3   __keep__z[motie_org] wipe failed: r   i  z%[motie_org] run_once completed: rows=z, pages=)r   requestsSessionr"   rangeextendrG   timesleeptabledeleteneqexecute	Exceptionr   warningr:   upsertinfo)
supabaser   r   all_rowspeinsertedbatchichunks
             r!   run_oncera   P   s   ||H				 t%%!q$(## 	 	AOOKa00111JsOOOO	              8{##**,,00DDLLNNNN 8 8 86166777777778 HE1c(mmU++  1u9% 	{##**51199;;;CJJ
KPPP$PPQQQOs+   ABBBAC( (
D2DD__main__)r   rO   typingr   r   rK   bs4r   app.services.supabase_servicer   r   BASEr   r   rL   r   r"   rG   ra   __name__ r#   r!   <module>ri      s   				                 < < < < < < < < %%%-	'8+ 	' 	' 	' 	' 	'4h& 4c 4d4j 4 4 4 4l#    : zHJJJJJ r#   