§
    ®eîiÜ!  ã                   ó  — d dl mZmZmZmZ ddlmZmZmZm	Z	m
Z
mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ d d	lZd
eee                  dee         fd„Zdeeef         deeef         fd„Zd	dœdee         dee         deeef         fd„Zdedee         de	fd„Z	 d dededee         de
fd„Zd	dœdededee         de
fd„Z	 d dededee         dee         dee         f
d„Zdedede fd„Z!dededeeef         fd„Z"d	S )!é    )ÚOptionalÚListÚDictÚAnyé   )ÚScrapeOptionsÚWebhookConfigÚDocumentÚBatchScrapeResponseÚBatchScrapeJobÚPaginationConfig)ÚAsyncHttpClient)Úprepare_scrape_options)Úhandle_response_error)Únormalize_document_input)Úvalidate_batch_urlsNÚ	data_listÚreturnc                 óž   — g }| pg D ]E}t          |t          ¦  «        r.t          |¦  «        }|                     t	          di |¤Ž¦  «         ŒF|S )N© )Ú
isinstanceÚdictr   Úappendr
   )r   Ú	documentsÚdocÚ
normalizeds       úb/home/div/.hermes/hermes-agent/venv/lib/python3.11/site-packages/firecrawl/v2/methods/aio/batch.pyÚ_parse_batch_scrape_documentsr   
   sb   € Ø "€IØˆ˜Bð 5ð 5ˆÝ�c�4Ñ Ô ð 	5Ý1°#Ñ6Ô6ˆJØ×Ò�XÐ3Ð3¨
Ð3Ð3Ñ4Ô4Ð4øØÐó    Úbodyc                 ó´  — |                       d¦  «        s#t          |                       dd¦  «        ¦  «        ‚|                       d¦  «        |                       dd¦  «        |                       dd¦  «        |                       d¦  «        |                       d	¦  «        |                       d
¦  «        t          |                       dg ¦  «        pg ¦  «        dœS )NÚsuccessÚerrorúUnknown error occurredÚstatusÚ	completedr   ÚtotalÚcreditsUsedÚ	expiresAtÚnextÚdata©r%   r&   r'   Úcredits_usedÚ
expires_atr*   r+   )ÚgetÚ	Exceptionr   )r    s    r   Ú#_parse_batch_scrape_status_responser1      sÁ   € Ø�8Š8�IÑÔð EÝ˜Ÿš Ð*BÑCÔCÑDÔDÐDð —(’(˜8Ñ$Ô$Ø—X’X˜k¨1Ñ-Ô-Ø—’˜' 1Ñ%Ô%ØŸš Ñ/Ô/Ø—h’h˜{Ñ+Ô+Ø—’˜Ñ Ô Ý-¨d¯hªh°v¸rÑ.BÔ.BÐ.HÀbÑIÔIðð ð r   )ÚoptionsÚurlsr2   c                óž  — | st          d¦  «        ‚t          d„ | D ¦   «         ¦  «        }d|i}|r&t          |¦  «        }|r|                     |¦  «         |                     d¦  «        x}�0t          |t          ¦  «        r|n|                     d¬¦  «        |d<   |                     d¦  «        x}�||d<   |                     d	¦  «        x}�||d
<   |                     d¦  «        x}�||d<   |                     d¦  «        x}�||d<   |                     d¦  «        x}�(t          |¦  «                             ¦   «         }|r||d<   |S )NzURLs list cannot be emptyc                 ód   — g | ]-}t          |t          ¦  «        r|                     ¦   «         n|‘Œ.S r   )r   ÚstrÚstrip)Ú.0Úus     r   ú
<listcomp>z_prepare.<locals>.<listcomp>%   s4   € Ð)_Ð)_Ð)_ÐUVµzÀ!ÅSÑ7IÔ7IÐ*P¨!¯'ª'©)¬)¨)ÈqÐ)_Ð)_Ð)_r   r3   ÚwebhookT)Úexclude_noneÚappend_to_idÚ
appendToIdÚignore_invalid_urlsÚignoreInvalidURLsÚmax_concurrencyÚmaxConcurrencyÚzero_data_retentionÚzeroDataRetentionÚintegration)	Ú
ValueErrorr   r   Úupdater/   r   r6   Ú
model_dumpr7   )	r3   r2   ÚkwargsÚvalidated_urlsÚpayloadÚoptsÚwÚvÚtrimmed_integrations	            r   Ú_preparerP   !   s~  € Øð 6ÝÐ4Ñ5Ô5Ð5å(Ð)_Ð)_ÐZ^Ð)_Ñ)_Ô)_Ñ`Ô`€NØ% ~Ð6€GØð !Ý% gÑ.Ô.ˆØð 	!Ø�NŠN˜4Ñ Ô Ð Ø�ZŠZ˜	Ñ"Ô"Ð"ˆÐ/Ý",¨QµÑ"4Ô"4ÐY˜Q˜Q¸!¿,º,ÐTX¸,Ñ:YÔ:Yˆ�	ÑØ�ZŠZ˜Ñ'Ô'Ð'ˆÐ4Ø !ˆ�ÑØ�ZŠZÐ-Ñ.Ô.Ð.ˆÐ;Ø'(ˆÐ#Ñ$Ø�ZŠZÐ)Ñ*Ô*Ð*ˆÐ7Ø$%ˆÐ Ñ!Ø�ZŠZÐ-Ñ.Ô.Ð.ˆÐ;Ø'(ˆÐ#Ñ$Ø�ZŠZ˜Ñ&Ô&Ð&ˆÐ3Ý! !™fœfŸlšl™nœnÐØð 	9Ø%8ˆG�MÑ"Ø€Nr   Úclientc              ‹   ó¼  K  — t          |fi |¤Ž}|                      d|¦  «        ƒ d {V —†}|j        dk    rt          |d¦  «         |                     ¦   «         }|                     d¦  «        s#t          |                     dd¦  «        ¦  «        ‚t          |                     d¦  «        |                     d¦  «        |                     d	¦  «        ¬
¦  «        S )Nz/v2/batch/scrapeé�  zstart batch scraper"   r#   r$   ÚidÚurlÚinvalidURLs)rT   rU   Úinvalid_urls)rP   ÚpostÚstatus_coder   Újsonr/   r0   r   )rQ   r3   rI   rK   Úresponser    s         r   Ústart_batch_scraper\   <   sØ   è è € Ý�tÐ&Ð&˜vÐ&Ð&€GØ—[’[Ð!3°WÑ=Ô=Ð=Ð=Ð=Ð=Ð=Ð=€HØÔ˜sÒ"Ð"Ý˜hÐ(<Ñ=Ô=Ð=Ø�=Š=‰?Œ?€DØ�8Š8�IÑÔð EÝ˜Ÿš Ð*BÑCÔCÑDÔDÐDÝ $§(¢(¨4¡.¤.°d·h²h¸u±o´oÐTX×T\ÒT\Ð]jÑTkÔTkÐlÑlÔlÐlr   Újob_idÚpagination_configc           	   ƒ   ó°  K  — |                       d|› �¦  «        ƒ d{V —†}|j        dk    rt          |d¦  «         |                     ¦   «         }t	          |¦  «        }|d         }|r|j        nd}|r&|d         rt          | |d         ||¦  «        ƒ d{V —†}t          |d         |d	         |d
         |d         |d         |s|d         nd|¬¦  «        S )al  
    Get the status of a batch scrape job.
    
    Args:
        client: Async HTTP client instance
        job_id: ID of the batch scrape job
        pagination_config: Optional configuration for pagination behavior
        
    Returns:
        BatchScrapeJob containing job status and data
        
    Raises:
        Exception: If the status check fails
    ú/v2/batch/scrape/NrS   zget batch scrape statusr+   Tr*   r%   r&   r'   r-   r.   r,   )r/   rY   r   rZ   r1   Úauto_paginateÚ_fetch_all_batch_pages_asyncr   )rQ   r]   r^   r[   r    rK   Údocsra   s           r   Úget_batch_scrape_statusrd   G   s0  è è € ð& —Z’ZÐ <°FÐ <Ð <Ñ=Ô=Ð=Ð=Ð=Ð=Ð=Ð=€HØÔ˜sÒ"Ð"Ý˜hÐ(AÑBÔBÐBØ�=Š=‰?Œ?€DÝ1°$Ñ7Ô7€GØ�6Œ?€Dð 8IÐRÐ%Ô3Ð3Èd€MØð 
˜ œð 
Ý1ØØ�FŒOØØñ	
ô 
ð 
ð 
ð 
ð 
ð 
ð 
ˆõ Ø�xÔ Ø˜+Ô&Ø�gÔØ˜^Ô,Ø˜<Ô(Ø$1Ð;ˆW�VŒ_ˆ_°tØðñ ô ð r   )Úrequest_timeoutÚnext_urlre   c          
   ƒ   ó<  K  — |                       ||¬¦  «        ƒ d{V —†}|j        dk    rt          |d¦  «         |                     ¦   «         }t	          |¦  «        }t          |d         |d         |d         |d         |d	         |d
         |d         ¬¦  «        S )a·  
    Fetch a single page of batch scrape results using the provided next URL.

    Args:
        client: Async HTTP client instance
        next_url: Opaque next URL from a prior batch scrape status response
        request_timeout: Timeout (in seconds) for the HTTP request

    Returns:
        BatchScrapeJob with the page data and next URL (if any)

    Raises:
        Exception: If the request fails or returns an error response
    )ÚtimeoutNrS   zget batch scrape status pager%   r&   r'   r-   r.   r*   r+   r,   )r/   rY   r   rZ   r1   r   )rQ   rf   re   r[   r    rK   s         r   Úget_batch_scrape_status_pageri   v   sµ   è è € ð( —Z’Z °/�ZÑBÔBÐBÐBÐBÐBÐBÐB€HØÔ˜sÒ"Ð"Ý˜hÐ(FÑGÔGÐGØ�=Š=‰?Œ?€DÝ1°$Ñ7Ô7€GÝØ�xÔ Ø˜+Ô&Ø�gÔØ˜^Ô,Ø˜<Ô(Ø�VŒ_Ø�VŒ_ðñ ô ð r   Úinitial_documentsc              ƒ   óÔ  K  — |                      ¦   «         }|}d}|r|j        nd}|r|j        nd}|r|j        nd}	t	          j        ¦   «         }
|�r|�||k    r�n|	�t	          j        ¦   «         |
z
  |	k    rnï|                      |¦  «        ƒ d{V —†}|j        dk    r7ddl}| 	                    d¦  «        }| 
                    d|j        › �¦  «         n’|                     ¦   «         }	 t          |¦  «        }n# t          $ r Y naw xY w|d         D ].}|�t          |¦  «        |k    r n|                     |¦  «         Œ/|�t          |¦  «        |k    rn|d         }|dz  }|�°|S )	ag  
    Fetch all pages of batch scrape results asynchronously.
    
    Args:
        client: Async HTTP client instance
        next_url: URL for the next page
        initial_documents: Documents from the first page
        pagination_config: Optional configuration for pagination limits
        
    Returns:
        List of all documents from all pages
    r   NrS   Ú	firecrawlzFailed to fetch next page: r+   r*   é   )ÚcopyÚ	max_pagesÚmax_resultsÚmax_wait_timeÚtimeÚ	monotonicr/   rY   ÚloggingÚ	getLoggerÚwarningrZ   r1   r0   Úlenr   )rQ   rf   rj   r^   r   Úcurrent_urlÚ
page_countro   rp   rq   Ú
start_timer[   rt   ÚloggerÚ	page_dataÚpage_payloadÚdocuments                    r   rb   rb   š   sä  è è € ð$ "×&Ò&Ñ(Ô(€IØ€KØ€Jð 0AÐJÐ!Ô+Ð+Àd€IØ3DÐNÐ#Ô/Ð/È$€KØ7HÐRÐ%Ô3Ð3Èd€Må”Ñ!Ô!€Jà
ñ %àÐ!¨
°iÒ(?Ð(?ÙàÐ%­D¬NÑ,<Ô,<¸zÑ,IÈ]Ò+ZÐ+ZØð  Ÿš KÑ0Ô0Ð0Ð0Ð0Ð0Ð0Ð0ˆàÔ 3Ò&Ð&àˆNˆNˆNØ×&Ò& {Ñ3Ô3ˆFØ�NŠNÐO¸Ô9MÐOÐOÑPÔPÐPØà—M’M‘O”Oˆ	ð	Ý>¸yÑIÔIˆLˆLøÝð 	ð 	ð 	ØˆEð	øøøð % VÔ,ð 	'ð 	'ˆHàÐ'­c°)©n¬nÀÒ.KÐ.KØ�Ø×Ò˜XÑ&Ô&Ð&Ð&ð Ð#­#¨i©.¬.¸KÒ*GÐ*GØð # 6Ô*ˆØ�a‰ˆ
ðK ñ %ðN Ðs   Ã+C; Ã;
DÄDc              ƒ   óÒ   K  — |                       d|› �¦  «        ƒ d {V —†}|j        dk    rt          |d¦  «         |                     ¦   «         }|                     d¦  «        dk    S )Nr`   rS   zcancel batch scraper%   Ú	cancelled)ÚdeleterY   r   rZ   r/   ©rQ   r]   r[   r    s       r   Úcancel_batch_scraperƒ   á   su   è è € Ø—]’]Ð#?°vÐ#?Ð#?Ñ@Ô@Ð@Ð@Ð@Ð@Ð@Ð@€HØÔ˜sÒ"Ð"Ý˜hÐ(=Ñ>Ô>Ð>Ø�=Š=‰?Œ?€DØ�8Š8�HÑÔ Ò,Ð,r   c              ƒ   ó  K  — |                       d|› d�¦  «        ƒ d {V —†}|j        dk    rt          |d¦  «         |                     ¦   «         }|                      d¦  «        s#t	          |                      dd¦  «        ¦  «        ‚|S )Nr`   z/errorsrS   zget batch scrape errorsr"   r#   r$   )r/   rY   r   rZ   r0   r‚   s       r   Úget_batch_scrape_errorsr…   é   s˜   è è € Ø—Z’ZÐ C°FÐ CÐ CÐ CÑDÔDÐDÐDÐDÐDÐDÐD€HØÔ˜sÒ"Ð"Ý˜hÐ(AÑBÔBÐBØ�=Š=‰?Œ?€DØ�8Š8�IÑÔð EÝ˜Ÿš Ð*BÑCÔCÑDÔDÐDØ€Kr   )N)#Útypingr   r   r   r   Útypesr   r	   r
   r   r   r   Úutils.http_client_asyncr   Úutils.validationr   Úutils.error_handlerr   Úutils.normalizer   Úmethods.batchr   rr   r   r6   r1   rP   r\   rd   Úfloatri   rb   Úboolrƒ   r…   r   r   r   ú<module>r�      s  ðØ ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rÐ rØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø €€€ð¨X°d¸3´iÔ-@ð ÀTÈ(Ä^ð ð ð ð ð¨d°3¸°8¬nð ÀÀcÈ3ÀhÄð ð ð ð ð EIð ð ð �4˜”9ð ¨(°=Ô*Að ÐW[Ð\_ÐadÐ\dÔWeð ð ð ð ð6m _ð m¸DÀ¼Ið mÐTgð mð mð mð mð 59ð,ð ,Øð,àð,ð  Ð 0Ô1ð,ð ð	,ð ,ð ,ð ,ðf (,ð	!ð !ð !Øð!àð!ð ˜e”_ð	!ð
 ð!ð !ð !ð !ðP 59ð	Dð DØðDàðDð ˜H”~ðDð  Ð 0Ô1ð	Dð
 
ˆ(„^ðDð Dð Dð DðN- oð -¸sð -Àtð -ð -ð -ð -ð¨/ð À3ð È4ÐPSÐUXÐPXÌ>ð ð ð ð ð ð r   