
    giF6                         S SK rS SKrS SKrS SKrS SKJr  / SQr/ SQrS\	S\
4S jrS\	S\4S	 jrS\	S\4S
 jrS\	S\	S\4S jrS\S\	S\4S jrSS jrS rS rS rS rS\	S\4S jrg)    N)async_playwright)z--no-sandboxz--disable-dev-shm-usagez--disable-gpuz--single-process)captchablockedzzugriff verweigertzaccess denied	forbiddenhtmlreturnc                    ^ U R                  5       m[        U4S j[         5       5      =(       a    [        U 5      S:  $ )Nc              3   ,   >#    U  H	  oT;   v   M     g 7f)N ).0mlcs     /app/autoscout24_playwright.py	<genexpr>"_is_bot_blocked.<locals>.<genexpr>   s     -1Bws   iP  )lowerany_BOT_MARKERSlen)r   r   s    @r   _is_bot_blockedr      s-    	B---C#d)e2CC    urlc                   #     [        U 5      I Sh  vN nU(       a  [        S[        U5       S35        U$ [        S5        [	        U 5      I Sh  vN $  NC! [         a  n[        SU S35         SnAN3SnAff = f N,7f)zJScrape AutoScout24 DE/NL via Playwright headless, fallback Firecrawl HTML.Nu   [AS24-Playwright] ✅ z	 annoncesu4   [AS24] Playwright 0 résultat — fallback Firecrawlz[AS24] Playwright erreur: u    — fallback Firecrawl)_playwright_searchprintr   	Exception_firecrawl_search)r   listingses      r   search_autoscout24r       s     G+C00*3x=/CDODE #3''' 1
  G*1#-DEFFG'sT   B A A$A B A B A>B A 
A;"A61B 6A;;B c                 
  #    [        5        IS h  vN nUR                  R                  S[        S9I S h  vN nUR	                  SSSS0S9I S h  vN nUR                  5       I S h  vN n UR                  U SS	S
9I S h  vN    UR                  SSS9I S h  vN   UR                  S5      I S h  vN    UR                  SSS9I S h  vN   UR                  5       I S h  vN n[        U5      (       a6  [        S5        / UR                  5       I S h  vN   sS S S 5      IS h  vN   $ UR                  S5      I S h  vN nU(       aU   [         R"                  " U5      n[%        Xp5      nU(       a+  UUR                  5       I S h  vN   sS S S 5      IS h  vN   $  ['        XP5      UR                  5       I S h  vN   sS S S 5      IS h  vN   $  GN GN GN GNz GNc GNM GN7! [         a     GNBf = f GN2! [         a     GN=f = f GN. N N N N N! [         a  n	[        SU	 35         S n	A	NS n	A	ff = f N Ns! UR                  5       I S h  vN    f = f! , IS h  vN  (       d  f       g = f7f)NT)headlessargszuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zde-DEzAccept-Languagezde-DE,de;q=0.9,en;q=0.8)
user_agentlocaleextra_http_headersdomcontentloadedi0u  )
wait_untiltimeoutzibutton[data-cy="uc-accept-all-button"], button:has-text("Alle akzeptieren"), #onetrust-accept-btn-handleri  )r)   i  zarticle[data-guid]i'  u,   [AS24-Playwright] page de blocage détectéez() => {
                const el = document.getElementById('__NEXT_DATA__');
                try { return el ? el.textContent : null; } catch(e) { return null; }
            }z-[AS24-Playwright] __NEXT_DATA__ parse error: )r   chromiumlaunch_LAUNCH_ARGSnew_contextnew_pagegotoclickwait_for_timeoutr   wait_for_selectorcontentr   r   closeevaluatejsonloads_parse_next_data_parse_as24_html)
r   pbrowsercontextpager   next_data_rawnditemsr   s
             r   r   r   %   ss    !!Q

))4l)KK++L 13LM	 , 
 
 %%'',	"))C,>)NNN	jj3 !	 !    ++D111
,,-A5,QQQ 'Dt$$DE& --/!!k "!!J #'-- 1 # M OM2B,R5E$ --/!!k "!!Z  $D.--/!!k "!!K
 ( O 2 
 R  (, "k "J  "k "^ ! OI!MNNO "k "j --/!!k "!!s  JGJ"I)GI)G!I)*G$+I)0IG'I
G0G*G06G-7G0<HHHI'H( II)HI) J,H-J2IHI)H"<I)HI)J H !J&I1I)II)	JI	JI)!I)$I)'I*G0-G00
G>:I=G>>IH
HIHII)JII) J"
I,H?:I?III)	JI&I"
 I&&I))J /I20J <Jbase_urlc                    SU;   nU(       a  SOSnU(       a  SOSn[         R                  " S[         R                  5      n/ n[        5       nUR	                  U 5       GHJ  nUR                  S5      n	[        [         R                  " SU	5      5      n
U
R                  S	S
5      nU(       a  X;   a  MU  U
R                  SS
5      S;  a  Mm  UR                  U5        U
R                  SS
5      n U(       a  US:w  a  [        U5      OS nUb  US:  a  M  U
R                  SS
5      n U(       a  US:w  a  [        U5      OS nU
R                  SS
5      nS nU(       a>  US;  a8  [         R                  " SU5      nU(       a  [        UR                  S5      5      nU SU SU 3nUR                  5       nU R                  SU5      nUU:  a  U UU OS
n[         R                  " SU5      n[         R                  " SU[         R                  5      nU(       a  UR                  S5      R                  5       OS
nS
nU(       aB  [         R                   " SS
UR                  S5      [         R                  S9R                  5       nU(       dU  U
R                  SS
5      R#                  5       nU
R                  SS
5      R#                  5       nU SU 3R                  5       n[$        R&                  " U SU 3R                  5       5      n[         R                   " SSU5      n[         R                  " SU5      nS nU(       a?  [         R                   " SS UR                  S5      5      n[         R                   " S!S"U5      nU(       d
  U(       d  GM2  UR)                  UXUUUS#S$.5        GMM     U$ ! [         a    S n GNf = f! [         a    S n GNf = f)%Nautoscout24.nlhttps://www.autoscout24.nlhttps://www.autoscout24.deaanbodangebotez<article\s([^>]{50,2000})>   z#data-([a-z\-]+)=["\']([^"\']+)["\']guid testid)z	list-itemrJ   priceunknown  mileagezfirst-registration)newrM   rJ   (\d{4})/z
</article>z-ListItemTitle_title[^"]*"[^>]*>([^<]+)</span>z.ListItemTitle_subtitle[^"]*"[^>]*>(.*?)</span>z
<!--.*?-->)flagsmakemodel z\s+zV(https://prod\.pictures\.autoscout24\.net/listing-images/[^\s"\']+\.(?:jpg|jpeg|webp))z/\d+x\d+\.webp$z/1200x900.jpgz/\d+x\d+\.(jpg|jpeg)$z/1200x900.\1EURtitleprixkmanneer   photodevise)recompileDOTALLsetfinditergroupdictfindallgetaddint
ValueErrorsearchendfindstripsub
capitalize	_html_libunescapeappend) r   rA   is_nloriginsegarticle_patternr   seenr   tagattrsrI   prix_rawrZ   km_rawr[   regr\   y_mitem_url	art_startart_endart_bodytitle_msub_m
main_titlesubtitlemkmorY   photo_mr]   s                                    r   r9   r9   m   sW   (E-2)8TF(C jj!>		JOH5D%%d+ggajRZZ FLMyy$t|99Xr"*;; 99Wb)	$,Y1F3x=DD t 9b)	 &6Y+>VDB
 ii,b1344))J,CCIIaL) XQse1TF+ EEG	))L)4.5	.A4	'*r))LhW		KXWYW`W`a18W]]1%++-b
vvmRQryyQWWYH 62&113B7B'224B4q++-J""j\8*#=#C#C#EFvsE* ))e
 FF-a@PQEFF3_eLEDDOOx% O ,Z O}  	D	  	B	s$   /N/'O/N?>N?OOdatac                 t   SU;   nU(       a  SOSnU(       a  SOSn[        U 5      nU(       d  / $ / nUS S  GH|  n[        U[        5      (       d  M  UR                  SS5      nU(       d3  UR                  S	UR                  S
S5      5      n	U	(       a
  U SU SU	 3nU(       a  UR	                  S5      (       d  X8-   nUR                  SS5      n
UR                  SS5      nUR                  SUR                  SS5      5      nSR                  [        S XU/5      5      R                  5       =(       d    UR                  SS5      n[        U5      n[        U5      n[        U5      n[        U5      nU(       d
  U(       d  GMO  U(       d  GMY  Ub	  US:  d  GMe  UR                  XUXUSS.5        GM     U$ )NrC   rD   rE   rF   rG      r   rJ   idrI   rR   httprT   rU   versionmodelVersionrV   rY   rN   rW   rX   )_find_listings_array
isinstancere   rg   
startswithjoinfilterrn   _extract_price_extract_year_extract_mileage_extract_photors   )r   rA   rt   ru   rv   r@   r   itemr   item_idrT   rU   r   rY   rZ   r\   r[   r]   s                     r   r8   r8      s   (E-2)8TF(C &E	Hcr
$%%88E2&hhtTXXfb%9:G$XQse1WI6H//77(Hxx#"%((9dhh~r&BCtG&<=>DDF_$((SZ\^J_d#d#d#t$TTxxT\TT\OOB% / 8 Or   c                    US:  d  [        U [        5      (       d  g S Hd  nU R                  U5      n[        U[        5      (       d  M+  U(       d  M4  [        US   [        5      (       d  MN  SUS   ;   d  SUS   ;   d  Mb  Us  $    U R	                  5        H3  n[        U[        5      (       d  M  [        XAS-   5      nU(       d  M1  Us  $    g )N   )classifiedsr   adsresultsr@   vehiclesr   r   rL   r   rH   )r   re   rg   listvaluesr   )objdepthkeyvalvrs         r   r   r      s    qy
3--Yggclc4  SSZA-E-E#a& DCFN
	 Z
 ZZ\a$Q	2Aq	 
 r   c                 p   U R                  S5      =(       d    U R                  S5      n[        U[        [        45      (       a  US:  a  [        U5      $ S $ [        U[        5      (       aJ  S HD  nUR                  U5      n[        U[        [        45      (       d  M1  US:  d  M9  [        U5      s  $    g )NrL   
priceValuer   )valueamountgrossnetrg   r   ri   floatre   )r   r:   kr   s       r   r   r     s    3TXXl3A!c5\""Qs1v(D(!T4AaA!c5\**q1u1v 5 r   c                    U R                  S5      =(       d)    U R                  S5      =(       d    U R                  S5      n[        U[        5      (       a  SUs=:  a  S:  a   U$   [        U[        5      (       a;  UR                  S5      =(       d    UR                  S5      nU(       a  [        U5      $ [        U[        5      (       a8  [
        R                  " SU5      nU(       a  [        UR                  S5      5      $ g )	NfirstRegistrationregistrationDateyeari  i  yrQ   rH   )rg   r   ri   re   strr_   rk   rd   )r   r}   r   r   s       r   r   r     s    
((&
'
[4884F+G
[488TZK[C#ss 1T 1
 !2#tGGFO+swws|q6M#sIIj#&qwwqz?"r   c                    U R                  S5      =(       d    U R                  S5      n[        U[        [        45      (       a  US:  a  [        U5      $ S $ [        U[        5      (       aU  UR                  S5      =(       d    UR                  S5      n[        U[        [        45      (       a  US:  a  [        U5      $ g )NmileageInKmrO   r   r   r   )r   r[   r   s      r   r   r     s    	-	 	7DHHY$7B"sEl##q&s2w*d*"dFF7O4rvvm4a#u&&1q5q6Mr   c                 
   S H  nU R                  U5      n[        U[        5      (       d  M+  U(       d  M4  US   n[        U[        5      (       a  UR	                  S5      (       a  Us  $ [        U[
        5      (       d  M  S H:  nUR                  US5      nU(       d  M  UR	                  S5      (       d  M6  Us  s  $    M     S H8  nU R                  US5      nU(       d  M  UR	                  S5      (       d  M6  Us  $    g )N)imagespicturesphotosmediar   r   )r   srcurihrefrJ   )imageUrlthumbnailUrlr]   image)rg   r   r   r   r   re   )r   r   imgsimgr   r   s         r   r   r   *  s    8xx}dD!!ddq'C#s##v(>(>
#t$$6A2AqQ\\&11  7 9 >HHS"1f%%H > r   c                 8  #     SSK Jn  [        R                  R	                  SS5      nU(       a  UR                  S5      (       a  [        S5        / $ U" US9nUR                  U S/S	9n[        US5      (       a  UR                  OSnU(       d)  [        U[        5      (       a  UR	                  SS5      OSnU(       d  [        S
5        / $ [        XP5      n[        S[        U5       S35        U$ ! [         a  n[        SU 35        / s SnA$ SnAff = f7f)uN   Fallback Firecrawl pour AutoScout24 — extrait le HTML et parse les articles.r   )	FirecrawlFIRECRAWL_API_KEYrJ   	REMPLACERu2   [AS24-Firecrawl] FIRECRAWL_API_KEY non configurée)api_keyr   )formatsz[AS24-Firecrawl] HTML videu   [AS24-Firecrawl] ✅ u    annonces parséesu   [AS24-Firecrawl] ❌ Erreur: N)	firecrawlr   osenvironrg   r   r   scrapehasattrr   r   re   r9   r   r   )r   r   r   fcresultr   r   r   s           r   r   r   A  s     '**..!4b9',,[99FGIw'31%ff55v{{2-7-E-E6::fb)2D./I#D.%c(m_4FGH -aS12	sN   DAC3 DA9C3 D$C3 2D3
D=DDDDD)r   )r   rq   r   r_   r6   playwright.async_apir   r,   r   r   boolr   r   r    r   r9   re   r8   r   r   r   r   r   r   r   r   r   <module>r      s     	 	  1
 ZD# D$ D

(# 
($ 
("6"# 6"$ 6"PW3 W# W$ W|&4 &3 &4 &R 	.  r   