h" d dl mZ d dlZd dlZd dlmZmZ d dlmZmZm Z m
Z
d dl Z d dl m
Z
d dlmZmZ d dlmZ e ZdZd Zd
d iZd ed
efdZdddedededededz f
dZded
e e
eeef fdZded
e
e e ef fdZ de e
d
e e fdZ d ed
e e fd Z d ed ed ed
e e fd!Z d"ed
e fd#Z!d$ed
e fd%Z"d
eeeez f fd&Z#e$d'k r e% e# dS dS )( )escapeN)datetimetimezone)DictListOptionalTuple)
BeautifulSoup)
get_clientlogger)broadcast_htmlz5https://www.moef.go.kr/nw/notice/hr.do?menuNo=4050300z,https://www.moef.go.kr/nw/notice/hrDetail.doz
User-AgentzCMozilla/5.0 (compatible; govbot/1.0; +https://work.jjickjjicks.com)targetreturnc dd l }t | } t d || dddd t
j t j
d n.# t $ r!}t j d| Y d }~nd }~ww xY w|S )Nr crawler_runrunning)idr statuspagesrows fail_reason
started_atz[crawler_run insert] )uuidstruuid4supabasetableinsertr nowr utc isoformatexecute Exceptionr warning)r r run_ides )/var/www/html/bot/app/crawler/moef_n8n.py
_run_startr( s KKK
F 4}%%,, ",x|44>>@@.
.
79999 4 4 42q22333333334 Ms A5B
C'CC)r r% r r r r c |t j t j ||d}|||d< t
d | d| d S # t $ r"}t j d| Y d }~d S d }~ww xY w)N)r finished_atr r r r r z[crawler_run finish] )
r r r r r! r r updateeqr" r# r r$ )r% r r r r payloadr& s r' _run_finishr. + s |HL11;;== G !,
4}%%,,W5588vFFNNPPPPP 4 4 42q223333333334s AB
B? B::B?href_jsc t j d| pd }|sdS | d | d | d fS )za
javascript:fn_egov_select('4050300','MOSFBBS_...','POST_ID') -> (menuNo, bbsId, postId)
z=fn_egov_select\('(\d+)','([A-Za-z0-9_]+)','([A-Za-z0-9_]+)'\) N )researchgroup)r/ ms r' _extract_js_paramsr9 < sU RT[Ta_abbA t 771::qwwqzz1771:: --
title_textc $ | sdS t j d| }|rN| d | d fS d| fS )uL
'[태그] 제목' -> ('태그','제목'), 아니면 (None, 원제)
)Nr1 z^\[(.+?)\]\s*(.+)$r2 r3 N)r5 matchstripr7 )r; r8 s r' _split_tag_from_titler? E s x
&
(8(8(:(:;;A 6wwqzz !!1771::#3#3#5#555 !!## ##r: containerc | sdS | d }|rD| dd pd }| d }|pdS dS )u~
목록 DOM에서 'state*' 클래스를 갖는 태그(예: 인사발령)를 우선 추출
Nz@span[class*="state"], em[class*="state"], strong[class*="state"] Tr> r1 z[]())
select_oneget_textr> )r@ elts r' _extract_tag_from_staterH P sq t ` a aB
[[D[
)
)
/R 6 6 8 8
GGFOOyD 4r: htmlc & t j d| }|st j d| }|sdS | d dd } t j |d }| d S # t $ r | d }d t | D } t j d
| d }| d cY S # t $ r Y Y dS w xY ww xY w) uj
상세 페이지에서 '등록일' 또는 날짜 패턴(YYYY.MM.DD / YYYY-MM-DD) -> 'YYYY-MM-DD'
u3 등록일[^0-9]*(\d{4}[.\-](\d{1,2})[.\-](\d{1,2}))z#(\d{4}[.\-](\d{1,2})[.\-](\d{1,2}))Nr2 .-z%Y-%m-%dc N g | ]"\ }}|d k r| d n|#S )r r3 )zfill).0ips r'
z0_parse_date_from_detail_html..l s2 III1q1uu!IIIr: ) r5 r6 r7 replacer strptimestrftimer# split enumeratejoin)rI r8 rawdtpartss r' _parse_date_from_detail_htmlr\ ] s( H$OOA D I>r: rowc 0 g dg dg dg dg}|D ]} fd|D } t d | dS # t $ r$}t j d| d | Y d
}~|d
}~ww xY wd S ) u<
tag/posted_at 유무와 무관하게 유연 삽입
)r r^ r_ titletag posted_at)r r^ r_ rz r| )r r^ r_ rz r{ r r^ r_ rz c * i | ]}|v || S r )rO krx s r'
z _safe_insert.. s$ 666aQ#XX3q6XXXr: rq Tzinsert retry with fields z failed: NF)r r r r" r# r r$ )rx fields_priorityfieldsru r& s ` r' _safe_insertr s
?>>777111*** O " M M66666666 M NN9 % % , ,T 2 2 : : < < <44 M M M NKvKKKK L L L L L L L L M 5s ?A%%
B/BBc
d} t | }d}d}d} t j t t d }| |dz
}t
|j d }| d }|D ]}| dd } t | }
|
s+|
\ } } }
| d
|
}t | rHd }| d }|r.| d }|r|
d
d }|s|
d
d pd}| d pE| d p0| d p| d p|j }t! | }t# | \ }}|s|}n|} t% | | |
}n3# t& $ r&}t) j d| d| d }Y d }~nd }~ww xY w|| |
|p|d}|r||d< |r||d< t- | }|st) j d| |dz
}t. d| d |
d | }|r7d t1 | d t1 | d d d!n&d"t1 | d d d!}d#| d$| d%} t3 | } |t5 | t6 rt7 | n| rdndz
}n1# t& $ r$}t) j d&| d| Y d }~nd }~ww xY wt9 j d' t= |d(||) t) j d*| d+| |||d(d,S # t& $ rU}t= |d-||tA | . t) j! d/| tA | |d-d0cY d }~S d }~ww xY w)1Nrq r rc )re rf r2 z html.parserz$a[href*="javascript:fn_egov_select"]hrefr1 rL h3arB TrC u 기재부 인사발령litrarticledivzfetch posted_at failed for z: r} r{ r| zinsert failed for z?searchBbsId1=z&searchNttId1=z&menuNo=z[(z) rz ][u! 기재부 인사발령입니다.
z
[자세히 보기]ztelegram send failed for g ?passed)r r r zMOEF run: inserted=z, sent=)insertedsentr% r aborted)r r r r zMOEF run aborted: )errorr% r )"r( rg rh LIST_URLrj rk r
rl rr r9 rw
find_previousfindrE find_parentparentrH r? rn r# r r$ r ri r r
isinstanceinttimesleepr. infor exception) r r% r r r rm soupanchorsr r/ rd r] r^ r_ ro r; r linkr@ r{ tag2 clean_titler| r& rx okheadmessagesent_ress r' run_oncer s
F
F
EH D`H L7B ? ? ?
QV]33++ DEE P P AeeFB''G'00F
$* !FE6 ((((F"6**
J &&B
@wws|| @!% s$ !?!?J
U ZZ4Z 8 8 TM, ?
G/ G*%M, *G//CM, 4r s ' ' ' ' ' ' ' ' . . . . . . . . . . . . < < < < < < < < 7 7 7 7 7 7 :<< B
;
W s s " _c 4 4 4 4 4C 4s 4QTW[Q[ 4 4 4 4". .sC}1E(F . . . . $c $eHSM34F.G $ $ $ $ x
'> 8C= s x} ,0c 0# 0s 0x} 0 0 0 0 d t ,gH$sC#I~& gH gH gH gHR z E((** r: