
    ulh                        d dl Z d dlZd dlZd dlmZmZ d dlZd dlmZ d dl	m
Z
  e j        dd                                          Z ej        ed            ej        d	          Zd
ZdZdZddiZdedz  defdZddZdee         fdZdededefdZdedee         fdZd dee         dedefdZd Zedk    r e e                       dS dS )!    N)ListDict)BeautifulSoup)
get_client	LOG_LEVELINFOz[%(levelname)s] %(message)s)levelformatmoef_org_syncmoef_orgzQhttps://www.moef.go.kr/mi/orgnzt/org.do?bbsId=MOSFBBS_000000000097&menuNo=9040100za[href*="orgId="]z
User-Agentz*GovBot/1.0 (+https://work.jjickjjicks.com)sreturnc                     | sdS d                     |                     dd                                                    S )N      )joinreplacesplit)r   s    2/var/www/html/bot/app/crawler/moef_org_sync_n8n.py_cleanr      s;     r88AIIh,,2244555    c                  B   t                      } 	 |                     t                                                                        dd                                           	 |                     t                                                                        dd                                           n# t          $ r Y nw xY wt          	                    dt                     d S # t          $ r,}t          
                    dt          |           Y d }~d S d }~ww xY w)Nname	__never__nullz	%s wiped.zwipe %s failed (continuing): %s)r   table
TABLE_NAMEdeleteneqexecuteis_	Exceptionloggerinfowarning)sbes     r   wipe_moef_orgr)      s   	BI
##%%))&+>>FFHHH	HHZ  ''))--ff==EEGGGG 	 	 	D	K,,,,, I I I8*aHHHHHHHHHIs=   AC( #AB6 5C( 6
C C( C#C( (
D2!DDc                     t          j        t          t          d          } |                                  t          | j        d          }t                      }g }|                    t                    D ]}|                    dd          
                                }|rd|vr1|                    dd          }|                    d          r|n|                    d	          sd|z   n|}||v r|                    |           |                    ||d
           t                              dt!          |                     |S )N   headerstimeouthtml.parserhrefr   zorgId=title/http)linkr1   zcollected %d org links)requestsgetLIST_URLHEADERSraise_for_statusr   textsetselect	SEL_LINKSstrip
startswithaddappendr$   r%   len)rsoupseenoutar0   r1   r4   s           r   fetch_org_linksrH   &   s8   Xw;;;A//D55DC[[## 	3 	3uuVR  &&(( 	xt++gr""s++fttT__U[E\E\1f#**bf4<<

D5112222
KK(#c((333Jr   name_rawposition_rawc                 H   t          |           }t          |          }|r|s|S |}t          j        d|          d                                         }d ||hD             }|D ])}|d|z   fD ]Q}|                    |          r:|d t          |                                        d          }t          |          }Rt          j        dt          j        |           dd|                              d          }t          j        d	t          j        |           d
d|                              d          }|                    |          r:|d t          |                                        d          }t          |          }+t          |          }|r|n|S )Nu   [ \t(\[/·\-]r   c                     g | ]}||S  rM   ).0cs     r   
<listcomp>z,strip_position_from_name.<locals>.<listcomp>D   s    ,,,!,!,,,r   r   u    -·/–—z\s*\(\s*z	\s*\)\s*$r   z\s*z\s*\([^)]+\)\s*$)	r   rer   r>   endswithrB   rstripsubescape)	rI   rJ   nporiginalbase
candidatescandsufs	            r   strip_position_from_namer]   :   s   xA|A A H 8$a((+1133D,,aY,,,J  #*% 	 	Czz# kS	kN))-881II F9ry9992qAAHHWWF;")D//;;;RCCJJ=YY ::d 	,SYYJ,&&}55Aq		Aq		A11xr   r4   c           	      J   |                      d          r| nd| z   }t          j        |t          d          }|                                 t          |j        d          }g }|                    d          D ]}|                    d          r|	                    d          }t          |          d	k     rBt          |d
                             d                    }t          |d                             d                    }t          |d                             d                    }	t          |d                             d                    }
t          |d                             d                    }|d                             d          }|rC|                    d          r.|d                             dd                                          }
t#          ||	          }|	}|s
|s|s|s|
s|                    ||||
|d           |S )Nr3   zhttps://www.moef.go.krr+   r,   r/   ztable trthtd   r   T)r>               za[href^="tel:"]r0   ztel:r   )
departmentr   positionphonetask)r?   r5   r6   r8   r9   r   r:   r<   findfind_allrB   r   get_text
select_onehas_attrr   r>   r]   rA   )r4   urlrC   rD   rowstrtdsrf   rI   rJ   rh   responsibilitytel_ar   rg   s                  r   parse_org_detailru   [   s   //&))
N$$/G$/NCS'2666A//DDkk*%%  774== 	kk$s88a<<Ad ; ;<<
Ad ; ;<<Ad ; ;<<Ad ; ;<<Ad ; ;<<A!!"344 	>U^^F++ 	>&M))&"55;;==E (,?? 	 	J 	. 	E 	$ "
 
 	 	 	 	 Kr     rp   chunkc                 *   t                      }d}t          dt          |           |          D ]c}| |||z            }|s|                    t                                        |                                           |t          |          z  }d|S )Nr   )r   rangerB   r   r   insertr!   )rp   rw   r'   totaliparts         r   bulk_insertr~      s    	BE1c$ii''  Aa%iK  	
##D))11333TLr   c            
          t                       t                      } | st                              d           dddS g }t	          | d          D ]i\  }}|d         }t          |          }t                              d|t          |           |t          |                     |                    |           j|st                              d           dddS t          |          }t                              d	t          |           d|dS )
Nzno org links found.MOEF_ORGr   )sourceinsertedrb   r4   z(%d/%d) %s -> %d rowszno rows parsed.z%s sync done: inserted=%d)
r)   rH   r$   r%   	enumerateru   rB   extendr~   r   )linksall_rowsr|   itemr4   rp   r   s          r   run_oncer      s   OOOE 5)***$!444HUA&&  4F|%%+QE

D#d))LLL 5%&&&$!4448$$H
KK+ZBBB h777r   __main__)r   N)rv   ) osloggingrQ   typingr   r   r5   bs4r   app.services.supabase_servicer   getenvupperr   basicConfig	getLoggerr$   r   r7   r=   r8   strr   r)   rH   r]   ru   intr~   r   __name__printrM   r   r   <module>r      s   				  				                4 4 4 4 4 4BIk6**0022	  ),I J J J J		?	+	+
9	E
F6cDj 6S 6 6 6 6
I 
I 
I 
Id    ( s  #  #        B&3 &4: & & & &P	 	d4j 	 	s 	 	 	 	8 8 8& z	E((** r   