oh d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm Z m
Z
d dl m Z d dl
mZ e j dd Z ej ed
ej d Zd Zd
ZdZdZdZdZ ej d Z ddiZ d Z de de fdZ ddde de d e!d e!d e dz f
d Z"d e dz de fd!Z#d"e de$e% fd#Z&de$e% fd$Z'd%e$e% de
e!e!f fd&Z(d' Z)e*d(k r e+ e) dS dS )) N)
BeautifulSoup)escape)ListTuple
get_client)broadcast_html LOG_LEVELINFOz[%(levelname)s] %(message)s)levelformat motie_n8nzhttps://www.motie.go.krzhttps://www.motie.go.kr/kor/article/ATCL6e90bb9de?mno=&pageIndex=1&rowPageC=0&displayAuthor=&searchCategory=3&schClear=on&startDtD=&endDtD=&searchCondition=1&searchKeyword=#ztd:nth-of-type(3)ztd:nth-of-type(5)za[href^="/attach/down"]za[onclick^="article.view"]z\d+z
User-Agentz*GovBot/1.0 (+https://work.jjickjjicks.com)c t S )Nr */var/www/html/bot/app/crawler/motie_n8n.pySBr " s <<r targetreturnc dd l }ddlm}m} t | } t d || dddd | |j
d n4# t $ r'}t d| Y d }~nd }~ww xY w|S )Nr datetimetimezone crawler_runrunning)idr statuspagesrows fail_reason
started_atz[crawler_run insert] )uuidr r struuid4r tableinsertnowutc isoformatexecute Exceptionloggerwarning)r r" r r run_ides r
_run_startr0 % s KKK++++++++
F 4
=!!(( ",,x|44>>@@*
*
79999 4 4 42q22333333334 Ms A8B( (
C2 CC)r r. r r r r c ddl m }m} || |j ||d}|||d< t d | d| d S # t $ r(}t d| Y d }~d S d }~ww xY w)Nr r )r finished_atr r r r r z[crawler_run finish] )
r r r' r( r) r r% updateeqr* r+ r, r- ) r. r r r r r r payloadr/ s r _run_finishr6 7 s ++++++++ ||HL11;;== G !,
4
=!!((1144T6BBJJLLLLL 4 4 42q223333333334s AB
C( C Csc X | sdS d | S )N )joinsplit)r7 s r _cleanr= I s) r 88AGGII r htmlc z t | d }| d }g }|D ]}| t }| t }| t
}| t }t |r| d nd } t |r| d nd }
|r/| d r|d
nd } |r/| d r|d
nd} t | }
|
r|
d nd }|r;| | |
| r | d r
t | z n| |d
|S ) Nz html.parserztable tbody trT)stripr9 hrefonclickr /)titledatefileUrl articleId)r select
select_one SEL_TITLESEL_DATESEL_FILESEL_LINKr= get_texthas_attrr@
RE_ARTICLE_IDsearchgroupappend
startswithBASE)r> soupr itemstrtitle_eldate_elfile_ellink_elrD rE file_urlrB m
article_ids r parse_list_htmlr` N s } - -D ;;' ( (DE ==++==**==**==**Hx((t(444bIIwFw''d'333BGG/6^7;K;KF;S;S^76?((***Z^18aW=M=Mi=X=Xa')$**,,,_a ) )#$.QWWQZZZ$
LL 19eh>Q>QRU>V>VeD8OO]e '
Lr c t j t t d } | t | j S )N )headerstimeout)requestsgetLIST_URLHEADERSraise_for_statusr` text)rs r fetch_and_extractrl j s: Xw;;;A 16 " ""r rW c t }d\ }}| D ]%}| d }|s| d d d| d }|j rt d| d}t |
rt | n|}| d pd } | d
pd }
|| d } |
r|
| d < | d
| |dz
}n8# t $ r+} t d
| d| Y d } ~ d } ~ ww xY wdt# | d| d}
t% |
}|t' |t rt | n|rdndz
}# t $ r+} t d| d| Y d } ~ d } ~ ww xY w||fS )N)r r rG motie_idr z/kor/article/ATCL6e90bb9de/z/view?rD r9 rE )r rD posted_atzinsert fail for z: u" 산업부 인사발령입니다.
[z ]
[자세히 보기]r znotify fail for )r rf r% rH r4 limitr* datarU r# isdigitintr@ r& r+ r, r- r r
isinstance)rW sbinsertednotifieditaidexist
detail_url aid_valuerD postedr5 r/ r> sents r upsert_to_supabase_and_notifyr o s B Hh ": ":ff[!! $$++D1144T3??EEaHHPPRR : DDDDD
#C 0 0 2 2;CHHH &B--//&&..&B--//"U33 *#)GK HHZ ' ' 0 0 8 8 : : : MHH NN8c88Q88 9 9 9 HHHH
=u
=
="
=
=
=
:!$''D Zc%:%:RD dAQPQ RHH : : : NN8c88Q88 9 9 9 9 9 9 9 9 : X s0 8?E88
F- F((F->>$s1vvS\]]]]]]]]^s& A B, !A
B, ,
D6AD D D__main__),osreloggingre bs4r r> r typingr r app.services.supabase_servicer app.services.telegram_senderr getenvupperr
basicConfig getLoggerr, rU rg rJ rK rL rM compilerP rh r r# r0 rt r6 r= listdictr` rl r r __name__printr r r r sq 4 4 4 4 4 4 7 7 7 7 7 7 BIk6 * * 0 0 2 2 ),I J J J J ; ' ' :
% (
6""
E
F
s s $ _c
4
4
4
4
4C
4s
4QTW[Q[
4
4
4
4$ cDj S
# $t* 8#4: # # # #
& d & c3h & & & & V^ ^ ^. z E((** r