
    z8jZ                         d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
mZ ddlmZ ddlmZ dZ ej"                  dej$                        Zd	Z G d
 de      ZdedefdZ G d de      Zy)u  Proveedor de datos REAL vía el endpoint público de *syndication* de X.

Usa https://syndication.twitter.com/srv/timeline-profile/screen-name/<handle>,
el mismo que usan las webs para incrustar timelines. Ventajas:
  - Sin login y sin API key  -> cero riesgo para tu cuenta, gratis.
  - Sin el token anti-bot 'x-client-transaction-id' que rompe a otros scrapers.
  - Trae engagement real (favorite/retweet/reply/quote_count) y fecha.

Limitaciones conocidas (gestionadas aquí):
  - X devuelve un CACHÉ ANTIGUO para algunas cuentas. Lo DETECTAMOS (campo `stale`
    en `last_meta`) e intentamos un 'cache-bust' para forzar datos en vivo.
  - RATE-LIMIT (HTTP 429) si se piden muchas cuentas muy rápido -> pausa
    configurable entre peticiones y reintentos con backoff.

Tras cada `fetch_user_tweets`, `self.last_meta[handle]` guarda metadatos útiles
para la UI: {'newest': datetime|None, 'stale': bool, 'n_raw': int}.

Solo usa la librería estándar (urllib, json, re, time); no añade dependencias.
    N)datetimetimezone)ListOptional)Tweet)TweetProviderzuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36zA<script id="__NEXT_DATA__" type="application/json">(.*?)</script>zAhttps://syndication.twitter.com/srv/timeline-profile/screen-name/c                       e Zd ZdZy)SyndicationErrorz;Error al obtener/parsear datos del endpoint de syndication.N)__name__
__module____qualname____doc__     agent/providers/syndication.pyr
   r
   *   s    Er   r
   valuereturnc                 .    t        j                  | d      S )Nz%a %b %d %H:%M:%S %z %Y)r   strptime)r   s    r   _parse_created_atr   .   s    U$=>>r   c                       e Zd ZddefdZddZdedefdZdedefd	Zd
edede	e
   fdZdd
edededede	e
   f
dZdee   defdZddededede	e
   fdZy)SyndicationProviderNcfgc                 l   |xs i }t        |j                  dd            | _        t        |j                  dd            | _        t	        |j                  dd            | _        t        |j                  dd            | _        t        |j                  d	d
            | _        d| _	        i | _
        y )Nsyndication_delay_seconds   syndication_timeout   syndication_retries   syndication_stale_after_days   syndication_cache_bustTg        )floatgetdelaytimeoutintretriesstale_after_daysbool
cache_bust_last_request	last_meta)selfr   s     r   __init__zSyndicationProvider.__init__4   s    iR377#>BC
SWW%:B?@377#8!<= %cgg.La&P Qsww'?FG r   r   c                     | j                   t        j                         | j                  z
  z
  }|dkD  rt        j                  |       t        j                         | _        y )Nr   )r&   time	monotonicr-   sleep)r/   waits     r   	_throttlezSyndicationProvider._throttle?   sD    zzT^^-0B0BBC!8JJt!^^-r   handlec                     t         |j                  d      z   }| j                  r&|dt        t	        j                         dz         z  }|S )N@z	?dnt=1&_=i  )	_BASE_URLlstripr,   r(   r2   )r/   r7   urls      r   
_build_urlzSyndicationProvider._build_urlE   sC    &--,,??Ys499;#56788C
r   r<   c                 j   t         j                  j                  |t        dd      }d }t	        | j
                  dz         D ]i  }| j                          	 t         j                  j                  || j                        5 }|j                         j                  dd      cd d d        c S  t#        d|       # 1 sw Y   nxY w# t         j                  j                  $ r}|}|j                  dk(  r9|| j
                  k  r*t        j                  | j                   |d	z   z         Y d }~t#        d
|j                   |j                  dk(  r	dz         |dz         |d }~wt$        $ rA}|}|| j
                  k  r%t        j                  | j                          Y d }~dY d }~jd }~ww xY w)Nz	text/html)z
User-AgentAccept)headers   )r'   zutf-8ignorei  r    zHTTP z (rate-limit) zsin respuesta tras reintentos: )urllibrequestRequest_UAranger)   r6   urlopenr'   readdecodeerror	HTTPErrorcoder2   r4   r&   r
   	Exception)r/   r<   reqlast_errattemptrespes          r   	_http_getzSyndicationProvider._http_getL   sp   nn$$SP[2\$]T\\A-.GNN^^++C+F$99;--gx@ GF /$ !@
KLL GFF<<)) 66S=Wt||%;JJtzzWq[9:&AFF8$166S=QNPQ  T\\)JJtzz* *sC   +C
 B="	C
=C	C

F2'AE%14E%%F210F--F2htmlc                 @   t         j                  |      }|st        d      t        j                  |j                  d            }	 |d   d   d   d   }g }|D ]x  }|j                  d      xs i j                  d	      }|rd
|vr/	 t        |d         }	|j                  d      xs i j                  d      xs |j                  d      }
|j                  d      }|j                  t        t        |d
         |
|j                  d      xs |j                  d      xs d|	t        |j                  d      xs d      t        |j                  d      xs d      t        |j                  d      xs d      t        |j                  d      xs d      t        |      j!                         rt        |      ndd|
 d|d
    
             { |j#                  d d       |S # t        t        f$ r t        d      w xY w# t        t        f$ r Y w xY w)zLTodos los tweets de la respuesta (orden descendente), SIN filtrar por fecha.u7   no se encontró __NEXT_DATA__ (¿X cambió el formato?)rA   props	pagePropstimelineentriesz%estructura inesperada en la respuestacontenttweetid_str
created_atuserscreen_namer9   
view_count	full_texttextrC   favorite_countr   retweet_countreply_countquote_countzhttps://x.com/z/status/)
idauthorrd   r_   likesretweetsrepliesquotesviewsr<   c                     | j                   S N)r_   )ts    r   <lambda>z0SyndicationProvider._parse_all.<locals>.<lambda>   s    q||r   T)keyreverse)
_NEXT_DATAsearchr
   jsonloadsgroupKeyError	TypeErrorr%   r   
ValueErrorr;   appendr   strr(   isdigitsort)r/   rV   r7   mdatar[   outentrytwcreatedrj   ro   s               r   
_parse_allzSyndicationProvider._parse_alld   s   d#"#\]]zz!''!*%	L7mK0<YGG E))I&,"11':B++B|,<= ffVn*//>T&--PSBTFFF<(EJJ2h<(!,DvD"&bff%56;!< !8!=A>} 5 :;rvvm49:(+E
(:(:(<#e*!(HG 0 	+T:
; )$ 	L"#JKK	L j) s   G, 
H
,H
HHsincelimitc                 r    | j                  ||      D cg c]  }|j                  |k\  s| c}d| S c c}w )z)Tweets dentro de la ventana [since, now].N)r   r_   )r/   rV   r7   r   r   rr   s         r   
parse_htmlzSyndicationProvider.parse_html   s;    ??48R8aALLE<Q8RSYTYZZRs   44newestc                     |yt        j                  t        j                        |z
  j                  | j
                  k\  S )NT)r   nowr   utcdaysr*   )r/   r   s     r   is_stalezSyndicationProvider.is_stale   s3    >X\\*V399T=R=RRRr   c                 P   | j                  | j                  |            }| j                  ||      }|r|d   j                  nd }|| j	                  |      t        |      d| j                  |j                  d      <   |D cg c]  }|j                  |k\  s| c}d | S c c}w )Nr   )r   stalen_rawr9   )rU   r=   r   r_   r   lenr.   r;   )r/   r7   r   r   rV   
all_tweetsr   rr   s           r   fetch_user_tweetsz%SyndicationProvider.fetch_user_tweets   s    ~~doof56__T62
-7A))T]]6*_.
v}}S)*
 &?:a)>:?GG?s   B#B#rq   )r   N)2   )r   r   r   dictr0   r6   r   r=   rU   r   r   r   r   r(   r   r   r+   r   r   r   r   r   r   r   3   s    D .  MS MS M0%s %C %DK %N[s [C [ [ [VZ[`Va [Sx1 Sd S
	H 	HH 	HS 	HRVW\R] 	Hr   r   )r   rx   rer2   urllib.errorrD   urllib.requestr   r   typingr   r   agent.modelsr   agent.providers.baser   rG   compileSrv   r:   rO   r
   r   r   r   r   r   r   <module>r      s   (  	    ' !  .L  RZZH"$$
 P	Fy F?S ?X ?
jH- jHr   