eik> UdZddlZddlZddlZddlmZmZmZmZm Z m Z m Z m Z m Z ddlZddlmZddlZddlZddlZddlZddlZddlZejdZejed<dZeZe dZGd d ejZGd d ejZGd dejZ GddejZ!Gddeje eZ"GddejZ#GddejZ$GddejZ%GddejZ&GddejZ'GddejZ(Gdd ejZ)Gd!d"ejZ*Gd#d$ejZ+Gd%d&ejZ,Gd'd(ejZ-Gd)d*ejZ.Gd+d,ejZ/Gd-d.ejZ0Gd/d0ejZ1Gd1d2e&Z2Gd3d4e"ee eZ3Gd5d6ejZ4Gd7d8ejZ5Gd9d:ejZ6Gd;dejZ8Gd?d@ejZ9GdAdBejZ:GdCdDejZ;GdEdFejZ<GdGdHejZ=GdIdJeje eZ>GdKdLejZ?GdMdNejZ@GdOdPejZAGdQdRejZBGdSdTejZCGdUdVejZDGdWdXejZEGdYdZejZFGd[d\ejZGGd]d^ejZHGd_d`ejZIGdadbejZJGdcddejZKGdedfejZLGdgdhejZMGdidjejZNGdkdlejZOGdmdNejZ@GdndHejZ=GdodpZPGdqdrZQGdsdtePZRGdudveQZSdS)wa" Firecrawl v1 API Client - Legacy Implementation This module provides the legacy v1 implementation of the Firecrawl SDK. It contains the complete `V1FirecrawlApp` class with all v1 API methods and types for backward compatibility. This is used by the unified client to provide version-specific access patterns like app.v1.scrape_url(). Classes: - V1FirecrawlApp: Legacy v1 client for interacting with the Firecrawl API. - AsyncV1FirecrawlApp: Async version of the v1 client. - CrawlWatcher: WebSocket-based crawl monitoring for v1. N) AnyDictOptionalListUnionCallableLiteralTypeVarGeneric)datetime firecrawlloggerc ddlm}tjt }|tj|d}tj d|tj }|r'| d SdS#t$rtdYdSwxYw)Nr)Pathz __init__.pyz"^__version__ = ['\"]([^'\"]*)['\"]z&Failed to get version from __init__.py)pathlibrospathdirname__file__join read_textresearchMgroupstrip Exceptionprint)r package_path version_file version_matchs W/home/div/.hermes/hermes-agent/venv/lib/python3.11/site-packages/firecrawl/v1/client.py get_versionr$s W__X..lT"',,|]CCDDNNPPli E|UWUYZZm 0$$Q''--// /00  4555 TTsB-B22CCTcHeZdZUdZdZeded<dZee ed<dS)V1AgentOptionszConfiguration for the agent.FIRE-1modelNprompt) __name__ __module__ __qualname____doc__r)r __annotations__r*rstrr#r'r'PsA&&'E78 ''' FHSM     r2r'c.eZdZUdZdZeded<dS)V1AgentOptionsExtract2Configuration for the agent in extract operations.r(r)Nr+r,r-r.r)r r/r1r2r#r4r4U-<<'E78 '''''r2r4c@eZdZUdZeeed<eeed<dS)V1ActionsResultz,Result of actions performed during scraping. screenshotspdfsN)r+r,r-r.rr0r/r1r2r#r9r9Ys366c s)OOOOOr2r9ceZdZUdZdZeeed<eed<eed<dZee ee fed<e j ddZ ee ed <dS) V1ChangeTrackingDataz. Data for the change tracking format. NpreviousScrapeAt changeStatus visibilitydiffjsonalias json_field)r+r,r-r.r>rr0r/rArrpydanticFieldrEr1r2r#r=r=^s'+hsm***OOO%)D(4S> "))) .t6 B B BJ BBBBBr2r=ceZdZUdZdZeeed<dZeeed<dZ eeed<dZ eeed<dZ ee eed<dZ eeed<ejdd Zeeed <dZeeed <dZeeed <dZeeed<dZeeed<dZeeed<dZeeed<dS)V1FirecrawlDocumentz-Document retrieved or processed by Firecrawl.NurlmarkdownhtmlrawHtmllinksextractrBrCrE screenshotmetadataactionstitle descriptionchangeTracking)r+r,r-r.rJrr0r/rKrLrMrNrrOr%rFrGrErPrQrrRr9rSrTrUr=r1r2r#rIrIhs577C#"Hhsm"""D(3-!GXc]!!!!%E8DI %%%GXa[,hnT@@@J @@@ $J $$$"Hhsm""")-GXo &---E8C=!%K#%%%59NH1299999r2rIcTeZdZUdZdZeeed<dZee eed<dS)V1LocationConfigz$Location configuration for scraping.Ncountry languages) r+r,r-r.rXrr0r/rYrr1r2r#rWrWxsE..!GXc]!!!%)IxS ")))))r2rWceZdZUdZeed<dZeeeefed<dZ eeeefed<dZ ee e ded<dS)V1WebhookConfigzConfiguration for webhooks.rJNheadersrQ) completedfailedpagestartedevents) r+r,r-r.r0r/r\rrrQrarr r1r2r#r[r[}s{%% HHH(,GXd38n %,,,)-HhtCH~&---PTFHT'"JKL MTTTTTr2r[ceZdZUdZdZeeeded<e j ddZ ee ed<dZ eeed<dZeeed <dS) V1ChangeTrackingOptionsz"Configuration for change tracking.N)zgit-diffrBmodesschemarC schema_fieldr*tag)r+r,r-r.rdrrr r/rFrGrfrr*r0rgr1r2r#rcrcs,,9=E8D!345 6==="0(.X"F"F"FL(3-FFF FHSM   C#r2rcceZdZUdZdZeeeded<dZ ee e e fed<dZ eee ed<dZ eee ed<dZeeed<dZeeed <d Zeeed <dZeeed <dZeeed <dZeeed<dZeeed<dZeeed<dZeeded<dZeeed<dZeeed<dZeeed<dZeeed<dS)V1ScrapeOptions#Parameters for scraping operations.N rKrLrMcontentrNrPscreenshot@fullPagerOrBrUformatsr\ includeTags excludeTagsonlyMainContentwaitFor0utimeoutlocationmobileskipTlsVerificationremoveBase64ImagesblockAdsbasicstealthenhancedautoproxychangeTrackingOptionsmaxAge storeInCacheparsePDF)r+r,r-r.rnrrr r/r\rr0rorprqboolrrintrtrurWrvrwrxryrrrcrrrr1r2r#riris--eiGXd7$_`abiii(,GXd38n %,,,'+K$s)$+++'+K$s)$+++&*OXd^***!GXc]!!!"GXc]"""+/Hh'(///!FHTN!!!*.$...)----#Hhtn###GKE8GBC DKKK?C8$;<CCC FHSM   #'L(4.'''#Hhtn#####r2ric^eZdZUdZeded<dZeeed<dZ ee ed<dS) V1WaitActionz'Wait action to perform during scraping.waittypeN millisecondsselector) r+r,r-r.r r/rrrrr0r1r2r#rrsP11 &/"&L(3-&&&"Hhsm"""""r2rc^eZdZUdZeded<dZeeed<dZ ee ed<dS)V1ScreenshotActionz-Screenshot action to perform during scraping.rPrNfullPagequality) r+r,r-r.r r/rrrrrr1r2r#rrsQ77 , #Hhtn###!GXc]!!!!!r2rc4eZdZUdZeded<eed<dS) V1ClickActionz(Click action to perform during scraping.clickrrNr+r,r-r.r r/r0r1r2r#rrs022 ' MMMMMr2rc4eZdZUdZeded<eed<dS) V1WriteActionz(Write action to perform during scraping.writertextNrr1r2r#rrs022 '  IIIIIr2rc4eZdZUdZeded<eed<dS) V1PressActionz(Press action to perform during scraping.pressrkeyNrr1r2r#rrs022 '  HHHHHr2rcZeZdZUdZeded<eded<dZeeed<dS)V1ScrollActionz)Scroll action to perform during scraping.scrollr)updown directionNr) r+r,r-r.r r/rrr0r1r2r#rrsN33 ( |$$$$"Hhsm"""""r2rc*eZdZUdZeded<dS)V1ScrapeActionz)Scrape action to perform during scraping.scraperN)r+r,r-r.r r/r1r2r#rrs)33 ( r2rc4eZdZUdZeded<eed<dS)V1ExecuteJavascriptActionz5Execute javascript action to perform during scraping.executeJavascriptrscriptNrr1r2r#rrs1?? % &&&& KKKKKr2rceZdZUdZeded<dZeeded<dZee ed<dZ ee ed<dS) V1PDFActionz&PDF action to perform during scraping.pdfrN) A0A1A2A3A4A5A6LetterLegalTabloidLedgerformat landscapescale) r+r,r-r.r r/rrrrrfloatr1r2r#rrsk00 %.rvFHWmn ovvv $Ix~$$$!E8E?!!!!!r2rc.eZdZUdZdZeded<dS)V1ExtractAgentr5r(r)Nr6r1r2r#rrr7r2rceZdZUdZdZeeed<ej ddZ ee ed<dZ eeed<dZ eeed<dS) V1JsonConfigzConfiguration for extraction.Nr*rerCrf systemPromptagent)r+r,r-r.r*rr0r/rFrGrfrrrrr1r2r#rrsz'' FHSM   "0(.X"F"F"FL(3-FFF"&L(3-&&&&*E8N #*****r2rc eZdZUdZdZeeed<dZeeed<dZ ee e e e eeeeeeef ed<dZeeed<dZeeed<dS)V1ScrapeParamsrjNrO jsonOptionsrRrwebhook)r+r,r-r.rOrrr/rrRrrrrrrrrrrrrr'rr[r1r2r#rrs--&*GXl #****.K,'...}AGXd5/A=R_anp~AOQjlw"wxyzAAA&*E8N #***)-GXo &-----r2rcVeZdZUdZdZeed<dZee ed<dZ ee ed<dS)V1ScrapeResponsez"Response from scraping operations.TsuccessNwarningerror) r+r,r-r.rrr/rrr0rr1r2r#rrsP,,GT!GXc]!!!E8C=r2rceZdZUdZdZeeed<dZeeed<dZ e ed<dZ eeed<dZ ee eed<dS) V1BatchScrapeResponsez&Response from batch scrape operations.NidrJTrr invalidURLs)r+r,r-r.rrr0r/rJrrrrrr1r2r#rrs}00B C#GTE8C='+K$s)$+++++r2rceZdZUdZdZeed<eded<eed<eed<eed<e ed <d Z e e ed <e eed <d S) V1BatchScrapeStatusResponsez)Response from batch scrape status checks.Trscrapingr]r^ cancelledstatusr]total creditsUsed expiresAtNnextdatar+r,r-r.rrr/r rr rrr0rrIr1r2r#rrs33GT B CCCCNNN JJJD(3- " ######r2rceZdZUdZdZeeeed<dZ eeeed<dZ ee ed<dZ ee ed<dZ ee ed<dZeeed<dZeeed <dZeeed <dZeeed <dZeeed <dZeeeefed <dZeeed<dZeeed<dZeeed<dZee ed<dZee ed<dZeeed<dS) V1CrawlParamsz#Parameters for crawling operations.N includePaths excludePathsmaxDepthmaxDiscoveryDepthlimitallowBackwardLinkscrawlEntireDomainallowExternalLinks ignoreSitemap scrapeOptionsrdeduplicateSimilarURLsignoreQueryParametersregexOnFullURLdelaymaxConcurrencyallowSubdomains)r+r,r-r.rrrr0r/rrrrrrrrrrrrirrr[rrrrrrr1r2r#rrs--(,L(49%,,,(,L(49%,,,"Hhsm"""'+x}+++E8C=)----(,x~,,,)----$(M8D>(((/3M8O,33359GXeC01 2999-1HTN111,08D>000%)NHTN)))E8C=$(NHSM(((&*OXd^*****r2rcpeZdZUdZdZeeed<dZeeed<dZ e ed<dZ eeed<dS)V1CrawlResponsez"Response from crawling operations.NrrJTrr) r+r,r-r.rrr0r/rJrrrr1r2r#rrsd,,B C#GTE8C=r2rceZdZUdZdZeed<eded<eed<eed<eed<e ed <d Z e e ed <e eed <d S) V1CrawlStatusResponsez"Response from crawl status checks.Trrrr]rrrNrrrr1r2r#rrs,,GT B CCCCNNN JJJD(3- " ######r2rcfeZdZUdZeed<dZeeed<eed<dZ eeed<eed<dS) V1CrawlErrorzA crawl error.rN timestamprJcoder) r+r,r-r.r0r/rrr rr1r2r#rr"sZ GGG$(Ix!((( HHHD(3- JJJJJr2rc@eZdZUdZeeed<eeed<dS)V1CrawlErrorsResponsez2Response from crawl/batch scrape error monitoring.errors robotsBlockedN)r+r,r-r.rrr/r0r1r2r#rr*s9<<  9r2rceZdZUdZdZeeed<dZee ed<dZ ee ed<dZ ee ed<dZ ee ed<dZee ed <dZee ed <dZeeed <dS) V1MapParamsz"Parameters for mapping operations.NrrincludeSubdomains sitemapOnlyrrsrtuseIndexru)r+r,r-r.rrr0r/rrrrrrrtrrurWr1r2r#rr/s,, FHSM   $(M8D>((((,x~,,,"&K$&&&E8C="GXc]"""#Hhtn###+/Hh'(/////r2rcbeZdZUdZdZeed<dZee e ed<dZ ee ed<dS) V1MapResponsez!Response from mapping operations.TrNrNr) r+r,r-r.rrr/rNrrr0rr1r2r#rr:sU++GT!%E8DI %%%E8C=r2rceZdZUdZdZeeed<ej ddZ ee ed<dZ eeed<dZ eeed<dZeeed <dZeeed <dZeeed <dZeeed <dZeeed <dS)V1ExtractParamsz0Parameters for extracting information from URLs.Nr*rerCrfrrenableWebSearchrorigin showSourcesr)r+r,r-r.r*rr0r/rFrGrfrrrrrrrrrrir1r2r#rr@s:: FHSM   "0(.X"F"F"FL(3-FFF"&L(3-&&&)----&*OXd^***(,x~,,, FHSM   "&K$&&&/3M8O,33333r2rceZdZUdZdZeeed<dZee ded<dZ ee ed<dZ e ed<dZeeed <dZeeed <dZeeed <dZeeeefed <dZeeed <dS)V1ExtractResponsez!Response from extract operations.Nr processingr]r^rrTrrrrsourcesr)r+r,r-r.rrr0r/rr rr rrrr%rrrrrrrr1r2r#rrLs++B EIFHW@A BIII$(Ix!(((GTD(1+E8C=!GXc]!!!(,GXd38n %,,,!%K#%%%%%r2rceZdZUeed<dZeeed<dZeeed<dZ eeed<dZ eeed<d Z eeed <dZ eeed <d Z eeed <dZeeed<dZeeed<dS)V1SearchParamsqueryrNtbsfilterenlangusrXruapiri`rtr)r+r,r-r0r/rrrr rrrXrurrtrrir1r2r#r r Xs JJJE8C=C# FHSM   D(3-!GXc]!!!"Hhsm"""!FHSM!!!"GXc]"""/3M8O,33333r2r cleZdZUdZdZeed<eeed<dZ e e ed<dZ e e ed<dS)V1SearchResponsez Response from search operations.TrrNrr) r+r,r-r.rrr/rrIrrr0rr1r2r#rrdsb**GT " ####!GXc]!!!E8C=r2rcheZdZUeed<dZeeed<dZeeed<dZ eeed<dS)V1CreditUsageDataremaining_creditsN plan_creditsbilling_period_startbilling_period_end) r+r,r-rr/rrrr0rr1r2r#rrks\"&L(3-&&&*.(3-...(, ,,,,,r2rc$eZdZUeed<eed<dS)V1CreditUsageResponserrN)r+r,r-rr/rr1r2r#rrqs' MMM r2rcheZdZUeed<dZeeed<dZeeed<dZ eeed<dS)V1TokenUsageDataremaining_tokensN plan_tokensrr) r+r,r-rr/r rrr0rr1r2r#rrus\!%K#%%%*.(3-...(, ,,,,,r2rc$eZdZUeed<eed<dS)V1TokenUsageResponserrN)r+r,r-rr/rr1r2r#r"r"{s' MMM r2r"cheZdZUdZeeed<dZeeed<dZeeed<e ed<dS)V1CreditUsageHistoricalPeriodN startDateendDateapiKeyr r+r,r-r%rr0r/r&r'rr1r2r#r$r$sZ#Ix}###!GXc]!!! FHSM   r2r$c0eZdZUeed<eeed<dS)V1CreditUsageHistoricalResponserperiodsN)r+r,r-rr/rr$r1r2r#r*r*s- MMM / 000000r2r*cheZdZUdZeeed<dZeeed<dZeeed<e ed<dS)V1TokenUsageHistoricalPeriodNr%r&r' tokensUsedr(r1r2r#r-r-sU#Ix}###!GXc]!!! FHSM   OOOOOr2r-c0eZdZUeed<eeed<dS)V1TokenUsageHistoricalResponserr+N)r+r,r-rr/rr-r1r2r#r0r0s- MMM . //////r2r0c|eZdZUdZdZeeed<dZee ed<dZ ee ed<dZ ee ed <dS) V1GenerateLLMsTextParamsz; Parameters for the LLMs.txt generation operation. maxUrlsF showFullTextTcacheN._V1GenerateLLMsTextParams__experimental_stream) r+r,r-r.r4rrr/r5rr6r7r1r2r#r2r2so GXc]#(L(4.((( E8D>   ,08D>00000r2r2ceZdZUdZdZeeed<dZeeed<dZ eeed<dZ ee ed <dZ ee ed <dZ eeed <dS) V1DeepResearchParamsz5 Parameters for the deep research operation. ri timeLimitr4NanalysisPromptr/_V1DeepResearchParams__experimental_streamSteps)r+r,r-r.rrrr/r;r4r=r0rr>rr1r2r#r9r9s Hhsm"Ix}"""GXc]$(NHSM((("&L(3-&&&1555555r2r9cBeZdZUdZeed<eed<dZeeed<dS)V1DeepResearchResponsez4 Response from the deep research operation. rrNr) r+r,r-r.rr/r0rrr1r2r#r@r@sDMMM GGGE8C=r2r@ceZdZUdZeed<dZeee e fed<e ed<dZ ee ed<e ed<e ed<e ed <e ee e fed <e ee e fed <e e ed <dS) V1DeepResearchStatusResponsez; Status response from the deep research operation. rNrrrr currentDepthr activitiesr summaries)r+r,r-r.rr/rrrr0rrrrr1r2r#rBrBsMMM%)D(4S> "))) KKKE8C=NNNMMMT#s(^$$$$ $sCx. !!!!Cyr2rBcFeZdZUdZdZeed<eed<dZe eed<dS)V1GenerateLLMsTextResponsez-Response from LLMs.txt generation operations.TrrNr) r+r,r-r.rrr/r0rrr1r2r#rGrGsE77GT GGGE8C=r2rGc4eZdZUeed<dZeeed<dS)$V1GenerateLLMsTextStatusResponseDatallmstxtN llmsfulltxt)r+r,r-r0r/rKrr1r2r#rIrIs/ LLL!%K#%%%%%r2rIcveZdZUdZdZeed<dZee ed<e ded<dZ ee ed<e ed <dS) V1GenerateLLMsTextStatusResponsez4Status response from LLMs.txt generation operations.TrNrrrrr) r+r,r-r.rrr/rrrIr rr0r1r2r#rMrMsk>>GT;?D(7 8??? 7 8888E8C=NNNNNr2rMcxeZdZUdZeed<eeee fed<dZ e eed<dZ e eed<dS)rz- Response from the search operation. rrNrr) r+r,r-r.rr/rrr0rrrrr1r2r#rrshMMM tCH~ !GXc]!!!E8C=r2ceZdZUdZdZeeed<ej ddZ ee ed<dZ eeed<dZ eeed <dZeeed <dZeeed <dZeeed <dZeeee fed <dS)rz/ Parameters for the extract operation. Nr*rerCrf system_promptFallow_external_linksenable_web_searchr show_sourcesr)r+r,r-r.r*rr0r/rFrGrfrrPrQrrRrrSrrr1r2r#rrs!FHSM   "0(.X"F"F"FL(3-FFF#'M8C='''+0(4.000(-x~---&+OXd^+++#(L(4.(((&*E8DcN #*****r2c0eZdZdZdZddeedeeddfdZdddddddddddddddddddddd d ed eee d d ee eefdeeedeeedee dee dee dee dee dee dee dee dee ddee deedeedeeeeeeeeeeeef deedee d ee d!ee deef.d"Zddddddddd#d$ed%ee d&eed'eed(eed)eedeedee d*eedefd+Zde fd,Z!de"fd-Z#dd/e de$fd0Z%dd/e de&fd1Z'ddddddddddddddddddd2dd3d ed4eeed5eeed6ee d7ee d%ee d8ee d9ee d:ee d;ee d*eedee d?ee d@ee dAee dBee d!ee dCee dDeede)f,dEZ*ddddddddddddddddddddFd ed4eeed5eeed6ee d7ee d%ee d8ee d9ee d:ee d;ee d*eedee d?ee d@ee dAee dBee d!ee dDeede+f*dGZ,dHede)fdIZ-dHede.fdJZ/dHede eeffdKZ0ddddddddddddddddddddFd ed4eeed5eeed6ee d7ee d%ee d8ee d9ee d:ee d;ee d*eedee d?ee d@ee dAee dBee d!ee dDeeddLf*dMZ1dddddddddNd edOeed;ee dPee dQee d%ee dee dRee dee de2fdSZ3dddddddddddddddddd2ddddTdUeed eee dVd ee eefdeeedeeedee dee dee dee dee dee dee dee dee ddeedeedeeeeeeeeeeeef dWee4dCee dBee d!ee dDeede5f.dXZ6dddddddddddddddddddddYdUeed eee dVd ee eefdeeedeeedee dee dee dee dee dee dee dee dee ddeedeedeeeeeeeeeeeef dWee4dBee dDeed!ee de7f,dZZ8ddddddddddddddddddddd[dUeed eee dVd ee eefdeeedeeedee dee dee dee dee dee dee dee dee ddeedeedeeeeeeeeeeeef dWee4dBee d!ee dDeeddLf,d\Z9dHede5fd]Z:dHede.fd^Z; ddddd.d.d.dd_dUeeed`eedaeedbeed:ee dcee ddee dWee eefde ddddd.d.d.dd_dUeeed`eedaeedbeed:ee dcee ddee dWee eefdez-V1FirecrawlApp.scrape_url..X(U(U(UCI*VT2J2J(}PVP[P[eix|P[P}P}(U(U(Ur2rRrrrzeroDataRetention /v1/scrape@@r r\rBrtrrrFailed to scrape URL. Error: +Failed to parse Firecrawl response as JSON.z scrape URLr1)_validate_kwargs_prepare_headersversionrz_ensure_schema_dictryupdaterequestspostrW status_coderBgetrrr^ _handle_error)r`rJrnr\rbrcrdrertrurvrfrgrhrrirOrjrRrkrlrmrnkwargs_headers scrape_paramsresponse response_jsons r#rpzV1FirecrawlApp.scrape_urls| fl333((**-G--   /'.M) $  /'.M) $  8+7M- (  8+7M- (  (/@M+ ,  0'/M) $  /'.M) $  X(0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  +%*M' "  (1M* %  ..w77G'4(( PX-@-@$($<$L>.L>>M)rr rrrXrurtscrape_optionsr rr rrrXrc 4|| di} ||| d<||| d<||| d<||| d<||| d<||| d<||| d <| | d d | d <| | | d } t dd|i| } | d d }dt |d<| r| |d <t j|jddd|j i|}|j dkr | }|drd|vr tdi|Sd|vrtd|dtd|#t$rtdwxYw||ddS)aS Search for content using Firecrawl. Args: query (str): Search query string limit (Optional[int]): Max results (default: 5) tbs (Optional[str]): Time filter (e.g. "qdr:d") filter (Optional[str]): Custom result filter lang (Optional[str]): Language code (default: "en") country (Optional[str]): Country code (default: "us") location (Optional[str]): Geo-targeting timeout (Optional[int]): Request timeout in milliseconds scrape_options (Optional[ScrapeOptions]): Result scraping configuration **kwargs: Additional keyword arguments for future compatibility Returns: SearchResponse: Response containing: * success (bool): Whether request succeeded * data (List[FirecrawlDocument]): Search results * warning (Optional[str]): Warning message if any * error (Optional[str]): Error message if any Raises: Exception: If search fails or response cannot be parsed rNrr rrrXrurtTrsr integrationr rqr /v1/search AuthorizationBearer r\rBrrrrzSearch failed. Error: rr1)rrzrrr rrrrWrVrrBrrr^r)r`r rr rrrXrurtrr search_params _integration final_params params_dictrrs r#rzV1FirecrawlApp.searchsTN fh///   %*M' " ?#&M%  &,M( #  $(M& !  '.M) $  (0M* %  '.M) $  %-;-@-@$]a-@-b-bM/ * V$$$$((77 &CCEC]CC "''D'II 7g 7 7 H  6)5K &=| ' ' '$&> &>&>?     3 & & O (   $$Y//NFm4K4K+<8E%7.E%%E?c|}||jd|}|jdkr |}|drd|vr t d i|Sd|vrtd|dtd|#t$rtdwxYw| |dd S) z1Get current credit usage and billing period (v1)./v1/team/credit-usagerrrrz#Failed to get credit usage. Error: rzget credit usageNr1) r _get_requestrWrrBrrrr^rr`rrrs r#get_credit_usagezV1FirecrawlApp.get_credit_usages((**$$| 2 2 2     3 & & O (   $$Y//[Fm4K4K0AA=AAA --#$b-X_J`$b$bccc#$Y-$Y$YZZZ O O O MNNN O   x); < < < < <8B&8.B&&Cc|}||jd|}|jdkr |}|drd|vr t d i|Sd|vrtd|dtd|#t$rtdwxYw| |dd S) z0Get current token usage and billing period (v1)./v1/team/token-usagerrrrz"Failed to get token usage. Error: rzget token usageNr1) rrrWrrBrr"rr^rrs r#get_token_usagezV1FirecrawlApp.get_token_usages((**$$| 1 1 1     3 & & O (   $$Y//ZFm4K4K/@@-@@@ --#$aW^I_$a$abbb#$X$X$XYYY O O O MNNN O   x): ; ; ; ; ;rF by_api_keyc|}|jd|rdndz}|||}|jdkr |}|drd|vr t d i|Sd|vrtd|dtd|#t$rtd wxYw| |d d S) z!Get historical credit usage (v1).z /v1/team/credit-usage/historical?byApiKey=truerrr+rz.Failed to get historical credit usage. Error: rzget credit usage historicalNr1) rrWrrrBrr*rr^rr`rrrJrrs r#get_credit_usage_historicalz*V1FirecrawlApp.get_credit_usage_historical3s.((**???WaCiCSCSgij$$S(33  3 & & O (   $$Y//fI4N4N:KK]KKK --#$mUbcjUk$m$mnnn#$dUb$d$deee O O O MNNN O   x)F G G G G G8B/.B//C c|}|jd|rdndz}|||}|jdkr |}|drd|vr t d i|Sd|vrtd|dtd|#t$rtd wxYw| |d d S) z Get historical token usage (v1).z/v1/team/token-usage/historicalrrrrr+rz-Failed to get historical token usage. Error: rzget token usage historicalNr1) rrWrrrBrr0rr^rrs r#get_token_usage_historicalz)V1FirecrawlApp.get_token_usage_historicalGs.((**>>>V`BhBRBRfhi$$S(33  3 & & O (   $$Y//eI4N4N9JJMJJJ --#$lTabiTj$l$lmmm#$cTa$c$cddd O O O MNNN O   x)E F F F F Fr) include_paths exclude_paths max_depthmax_discovery_depthrallow_backward_linkscrawl_entire_domainrQignore_sitemaprrdeduplicate_similar_urlsignore_query_parametersregex_on_full_urlrallow_subdomainsmax_concurrencyrn poll_intervalidempotency_keyrrrrrrrQrrrrrrrrrrc |||di}|||d<|||d<|||d<|||d<|||d<|||d<n|||d <| | |d <| | |d <| | d d |d<| | |d<| | |d<|||d<|||d<|||d<|||d<|||d<|||d<|||d}t d i|}|d d }||d<dt |d<|r||d<||}||jd||}|j dkrT | d}n#tdxYw| |||S| |ddS)!a Crawl a website starting from a URL. Args: url (str): Target URL to start crawling from include_paths (Optional[List[str]]): Patterns of URLs to include exclude_paths (Optional[List[str]]): Patterns of URLs to exclude max_depth (Optional[int]): Maximum crawl depth max_discovery_depth (Optional[int]): Maximum depth for finding new URLs limit (Optional[int]): Maximum pages to crawl allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead crawl_entire_domain (Optional[bool]): Follow parent directory links allow_external_links (Optional[bool]): Follow external domain links ignore_sitemap (Optional[bool]): Skip sitemap.xml processing scrape_options (Optional[ScrapeOptions]): Page scraping configuration webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings deduplicate_similar_urls (Optional[bool]): Remove similar URLs ignore_query_parameters (Optional[bool]): Ignore URL parameters regex_on_full_url (Optional[bool]): Apply regex to full URLs delay (Optional[int]): Delay in seconds between scrapes allow_subdomains (Optional[bool]): Follow subdomains max_concurrency (Optional[int]): Maximum number of concurrent scrapes zero_data_retention (Optional[bool]): Whether to delete data after 24 hours poll_interval (Optional[int]): Seconds between status checks (default: 2) idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: CrawlStatusResponse with: * Crawling status and progress * Crawled page contents * Success/error information Raises: Exception: If crawl fails crawl_urlNrrrrrrrrrTrsrrrrrrrrrrrJrqr /v1/crawlrrrstart crawl jobr1)rrzrrrrr _post_requestrWrrBr_monitor_job_statusr)r`rJrrrrrrrrQrrrrrrrrrrnrrr crawl_paramsrrrr\rrs r#rzV1FirecrawlApp.crawl_url[s~ fk222   $+8L (  $+8L (  '0L $  *0CL, -  $)L !  *0CL, - - ! -1EL- .  +1EL- .  %,:L )  %,:,?,?\`,?,a,aL )  &-L # # /5ML1 2 " .4KL0 1  (->L) *  $)L !  '.>L* +  &-L) *  $)L !  '.>L* +  &- > > > >=AA,AE6!D65E66E.E66 F'F""F'c|}||jd|d|}|jdkr3 t di|S#t dxYw||ddS)aL Returns information about crawl errors. Args: id (str): The ID of the crawl job Returns: V1CrawlErrorsResponse containing: * errors (List[Dict[str, str]]): List of errors with fields: - id (str): Error ID - timestamp (str): When the error occurred - url (str): URL that caused the error - error (str): Error message * robotsBlocked (List[str]): List of URLs blocked by robots.txt Raises: Exception: If error check fails r/errorsrrzcheck crawl errorsNr1rrrWrrrBrrr`rr\rs r#check_crawl_errorsz!V1FirecrawlApp.check_crawl_errorss&''))$$ %K%K%K%K%KWUU  3 & & P,??x}}??? P NOOO   x)= > > > > > A A1c|}||jd||}|jdkr) |S#t dxYw||ddS)} Cancel an asynchronous crawl job. Args: id (str): The ID of the crawl job to cancel Returns: Dict[str, Any] containing: * success (bool): Whether cancellation was successful * error (str, optional): Error message if cancellation failed Raises: Exception: If cancellation fails rrrzcancel crawl jobN)r_delete_requestrWrrBrrrs r# cancel_crawlzV1FirecrawlApp.cancel_crawls''))''4<(G(G2(G(GQQ  3 & & P}}& P NOOO   x); < < < <  *> G &>%= %<$; 0/ %! ".-# $,O% &!4 3' (,O+  .  ! 9n&7 9!."3T:: :788 8r2)rrinclude_subdomains sitemap_onlyrrt use_indexrurrrrc :|| di} ||| d<||| d<||| d<||| d<||| d<||| d<||| d <| | d d | d <| | | d } t di| } | d d }||d<dt |d<| r| |d <t j|jddd|j i|}|j dkr | }|drd|vr tdi|Sd|vrtd|dtd|#t$rtdwxYw||ddS)a Map and discover links from a URL. Args: url (str): Target URL to map search (Optional[str]): Filter pattern for URLs ignore_sitemap (Optional[bool]): Skip sitemap.xml processing include_subdomains (Optional[bool]): Include subdomain links sitemap_only (Optional[bool]): Only use sitemap.xml limit (Optional[int]): Maximum URLs to return timeout (Optional[int]): Request timeout in milliseconds **kwargs: Additional parameters to pass to the API Returns: V1MapResponse: Response containing: * success (bool): Whether request succeeded * links (List[str]): Discovered URLs * error (Optional[str]): Error message if any Raises: Exception: If mapping fails or response cannot be parsed map_urlNrrrrrrtrTrsrurrJrqr/v1/maprrrrrrNrzMap failed. Error: rmapr1)rrzrrrrrrrWrVrrBrrr^r)r`rJrrrrrrtrrur map_paramsrrrrrs r#rzV1FirecrawlApp.map_url7sUH fi000   #)Jx  %*8J '  ).@J* +  #(4J} %  "'Jw   $+Jy !  %.Jz "  %-]]Dt]%T%TJz " &!!!!~~m44 #00Z00 "''D'II  E 7g 7 7 H  6)5K &=| $ $ $$&> &>&>?     3 & & O (   $$Y//KG}4L4L(99=999 --#$R-:P$R$RSSS#$I-$I$IJJJ O O O MNNN O   x / / / / /s8E(:.E((F)rnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrrnrurls rKrLrMrlrNrPrmrOrBrc n||di}|||d<|||d<|||d<|||d<|||d<|||d<|||d <| | d d |d <| | |d <| | |d<| | |d<| | |d<|||d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|rd|D|d<||d d |d<|||d<|||d<||t d"i|}|d d }||d<dt |d<d|vr<|dr4d|dvr*||dd|dd<d|vr<|dr4d|dvr*||dd|dd<||}||j d||}|j dkrT | d}n#td xYw||||S||d!dS)#aH Batch scrape multiple URLs and monitor until completion. Args: urls (List[str]): URLs to scrape formats (Optional[List[Literal]]): Content formats to retrieve headers (Optional[Dict[str, str]]): Custom HTTP headers include_tags (Optional[List[str]]): HTML tags to include exclude_tags (Optional[List[str]]): HTML tags to exclude only_main_content (Optional[bool]): Extract main content only wait_for (Optional[int]): Wait time in milliseconds timeout (Optional[int]): Request timeout in milliseconds location (Optional[LocationConfig]): Location configuration mobile (Optional[bool]): Use mobile user agent skip_tls_verification (Optional[bool]): Skip TLS verification remove_base64_images (Optional[bool]): Remove base64 encoded images block_ads (Optional[bool]): Block advertisements proxy (Optional[Literal]): Proxy type to use extract (Optional[JsonConfig]): Content extraction config json_options (Optional[JsonConfig]): JSON extraction config actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration max_concurrency (Optional[int]): Maximum number of concurrent scrapes poll_interval (Optional[int]): Seconds between status checks (default: 2) idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: V1BatchScrapeStatusResponse with: * Scraping status and progress * Scraped content for each URL * Success/error information Raises: Exception: If batch scrape fails batch_scrape_urlsNrnr\rorprqrrrtTrsrurvrwrxryrrerOrcjg|]0}t|tr|n|dd1Srwrxr{s r#r~z4V1FirecrawlApp.batch_scrape_urls..rr2rRrrrrrqr/v1/batch/scraperrrstart batch scrape jobr1)rrzrryrrrrrrWrrBrrrr)r`rrnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrrnrrrrrrrs r#rz V1FirecrawlApp.batch_scrape_urlssM@ f&9:::   '.M) $  '.M) $  #+7M- (  #+7M- (  (/@M+ ,  '/M) $  '.M) $  (0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  %*M' "  ..w77G'4(( PX-@-@$($<$ #8{[dOeCeCe/3/G/G T]H^_gHh/i/iK "8 , K ' 'K ,F '8WbcpWqKqKq373K3KKXeLfgoLp3q3qK &x 0''88%%&G&G&GV]^^  3 & & P]]__((.. P NOOO++BGG G   x)A B B B B Bs 'K44L)rnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrrnc ,||di}|||d<|||d<|||d<|||d<|||d<|||d<|||d <| | d d |d <| | |d <| | |d<| | |d<| | |d<|||d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|rd|D|d<||d d |d<|||d<|||d<||t d!i|}|d d }||d<dt |d<d|vr<|dr4d|dvr*||dd|dd<d|vr<|dr4d|dvr*||dd|dd<||}||j d||}|j dkr3 td!i| S#tdxYw||d dS)"a~ Initiate a batch scrape job asynchronously. Args: urls (List[str]): URLs to scrape formats (Optional[List[Literal]]): Content formats to retrieve headers (Optional[Dict[str, str]]): Custom HTTP headers include_tags (Optional[List[str]]): HTML tags to include exclude_tags (Optional[List[str]]): HTML tags to exclude only_main_content (Optional[bool]): Extract main content only wait_for (Optional[int]): Wait time in milliseconds timeout (Optional[int]): Request timeout in milliseconds location (Optional[LocationConfig]): Location configuration mobile (Optional[bool]): Use mobile user agent skip_tls_verification (Optional[bool]): Skip TLS verification remove_base64_images (Optional[bool]): Remove base64 encoded images block_ads (Optional[bool]): Block advertisements proxy (Optional[Literal]): Proxy type to use extract (Optional[JsonConfig]): Content extraction config json_options (Optional[JsonConfig]): JSON extraction config actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration max_concurrency (Optional[int]): Maximum number of concurrent scrapes zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: V1BatchScrapeResponse with: * success - Whether job started successfully * id - Unique identifier for the job * url - Status check URL * error - Error message if start failed Raises: Exception: If job initiation fails async_batch_scrape_urlsNrnr\rorprqrrrtTrsrurvrwrxryrrerOrcjg|]0}t|tr|n|dd1Srwrxr{s r#r~z:V1FirecrawlApp.async_batch_scrape_urls..rr2rRrrrrrqrrrrrr1)rrzrryrrrrrrWrrrBrr)r`rrnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrrnrrrrrs r#rz&V1FirecrawlApp.async_batch_scrape_urls!s4@ f&?@@@   '.M) $  '.M) $  #+7M- (  #+7M- (  (/@M+ ,  '/M) $  '.M) $  (0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  %*M' "  ..w77G'4(( PX-@-@$($<$ #8{[dOeCeCe/3/G/G T]H^_gHh/i/iK "8 , K ' 'K ,F '8WbcpWqKqKq373K3KKXeLfgoLp3q3qK &x 0''88%%&G&G&GV]^^  3 & & P,??x}}??? P NOOO   x)A B B B B Bs K**K;)rnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrnrc ||di}|||d<|||d<|||d<|||d<|||d<|||d<|||d <| | d d |d <| | |d <| | |d<| | |d<| | |d<|||d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|rd|D|d<||d d |d<|||d<|||d<||t d"i|}|d d }||d<dt |d<d|vr<|dr4d|dvr*||dd|dd<d|vr<|dr4d|dvr*||dd|dd<||}||j d||}|j dkre td"i| }|j r|jrt|j|St!d#t!d xYw||d!dS)#a  Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket. Args: urls (List[str]): URLs to scrape formats (Optional[List[Literal]]): Content formats to retrieve headers (Optional[Dict[str, str]]): Custom HTTP headers include_tags (Optional[List[str]]): HTML tags to include exclude_tags (Optional[List[str]]): HTML tags to exclude only_main_content (Optional[bool]): Extract main content only wait_for (Optional[int]): Wait time in milliseconds timeout (Optional[int]): Request timeout in milliseconds location (Optional[LocationConfig]): Location configuration mobile (Optional[bool]): Use mobile user agent skip_tls_verification (Optional[bool]): Skip TLS verification remove_base64_images (Optional[bool]): Remove base64 encoded images block_ads (Optional[bool]): Block advertisements proxy (Optional[Literal]): Proxy type to use extract (Optional[JsonConfig]): Content extraction config json_options (Optional[JsonConfig]): JSON extraction config actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration max_concurrency (Optional[int]): Maximum number of concurrent scrapes zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: V1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket Raises: Exception: If batch scrape job fails to start batch_scrape_urls_and_watchNrnr\rorprqrrrtTrsrurvrwrxryrrerOrcjg|]0}t|tr|n|dd1Srwrxr{s r#r~z>V1FirecrawlApp.batch_scrape_urls_and_watch..rr2rRrrrrrqrrr Batch scrape job failed to startrrr1)rrzrryrrrrrrWrrrBrrrrr)r`rrnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrnrrrrrrrs r#rz*V1FirecrawlApp.batch_scrape_urls_and_watchslx f&CDDD   '.M) $  '.M) $  #+7M- (  #+7M- (  (/@M+ ,  '/M) $  '.M) $  (0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  %*M' "  ..w77G'4(( PX-@-@$($<$ #8{[dOeCeCe/3/G/G T]H^_gHh/i/iK "8 , K ' 'K ,F '8WbcpWqKqKq373K3KKXeLfgoLp3q3qK &x 0''88%%&G&G&GV]^^  3 & & P!6!I!I!I!I!)Hn.?H).*;TBBB#$FGGG P NOOO   x)A B B B B Bs AL LL-cd|}|}||j||}|jdkr# |}n#t dxYw|ddkr8d|vr3|d}d|vr!t |ddkrn|d}|st d n |||}|jdkr#t d |jn |} n#t dxYw| | dg| }n4#t $r'} t d | Yd } ~ n d } ~ wwxYwd|v!||d<tdid |vrdnd|d|d|d|d|d|d|d|d d S| |dd S)a@ Check the status of a batch scrape job using the Firecrawl API. Args: id (str): The ID of the batch scrape job. Returns: V1BatchScrapeStatusResponse: The status of the batch scrape job. Raises: Exception: If the status check request fails. /v1/batch/scrape/rrrr]rrrrrrNrFTrrr) rrrr]rrrrrzcheck batch scrape statusr1)rrrWrrBrrrrrrrrrrs r#check_batch_scrape_statusz(V1FirecrawlApp.check_batch_scrape_status9s,r++''))$$ %@h%@%@'JJ  3 & & P&mmoo  P NOOO8$ 33[((&v.D K//{6233q88!#.??6#:#:'""NN+LMMM! ".2.?.?'.R.RO.:cAA & -h?Kf-h-h i i i %`,;,@,@,B,B `&/0^&_&_ _ KK fb(A(ABBB*3KK(""""LL)PQ)P)PQQQ!EEEE"%!K//*+/K'.  $+{$:$:55%//(33$11(__[99*}==(__[99#//#//$11 2 2      x)D E E E E Erc|}||jd|d|}|jdkr3 t di|S#t dxYw||ddS)aX Returns information about batch scrape errors. Args: id (str): The ID of the crawl job. Returns: V1CrawlErrorsResponse containing: * errors (List[Dict[str, str]]): List of errors with fields: * id (str): Error ID * timestamp (str): When the error occurred * url (str): URL that caused the error * error (str): Error message * robotsBlocked (List[str]): List of URLs blocked by robots.txt Raises: Exception: If the error check request fails rrrrzcheck batch scrape errorsNr1rrs r#check_batch_scrape_errorsz(V1FirecrawlApp.check_batch_scrape_errorsws&''))$$ %R%Rr%R%R%RT[\\  3 & & P,??x}}??? P NOOO   x)D E E E E Err*rerPrQrRrSrr*rerPrRrSc ddl} | dtd|| d|} |s|st d|s|st d|r||}|pg||||d td } |r|| d <|r|| d <|r|| d <| |  | |j d| | } | j dkr5 | }n#tdxYw|dr|d}|std ||j d|| }|j dkrn | }n#tdxYw|ddkr t!d"i|S|ddvr!td|dd|dn||dt%jdtd|d|| dn/#t$r"}t t)|dd}~wwxYwt!dd !S)#a Extract structured information from URLs. .. deprecated:: The extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement. Args: urls (Optional[List[str]]): URLs to extract from prompt (Optional[str]): Custom extraction prompt schema (Optional[Any]): JSON schema/Pydantic model system_prompt (Optional[str]): System context allow_external_links (Optional[bool]): Follow external links enable_web_search (Optional[bool]): Enable web search show_sources (Optional[bool]): Include source URLs agent (Optional[Dict[str, Any]]): Agent configuration **kwargs: Additional parameters to pass to the API Returns: V1ExtractResponse[Any] with: * success (bool): Whether request succeeded * data (Optional[Any]): Extracted data matching schema * error (Optional[str]): Error message if any Raises: ValueError: If prompt/schema missing or extraction fails rNThe extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.r stacklevelrO#Either prompt or schema is required!Either urls or prompt is requiredrqrrrrrerr*rr /v1/extractrrrr)Job ID not returned from extract request.T /v1/extract/rr]r^r Extract job . Error: rzextract-statusFailed to extract. Error: FzInternal server error.rrr1)warningswarnDeprecationWarningrrr^rr$rrrWrrBrrrrrtimesleepr0)r`rr*rerPrQrRrSrrrr\ request_datarrjob_idrrrs r#rOzV1FirecrawlApp.extracts\P   %      fi000'')) Df DBCC C BF B@AA A  6--f55FJB"60'3KMM33     ,%+L "  9+8L (  *$)L ! F###) *))<,,,H #s**T#==??DDT#$RSSS ?R!XXd^^F!U'(STTT&*.*;*;#|AAAA#+++6#==`.=.B.B.D.D `&/0^&_&_ _*84 CC'8'G'G;'G'G G!,X!6:Q!Q!Q&/0u{8?T0u0u_jkr_s0u0u&v&v v"R!..@PQQQ 1 #&&$$Pg$P$PQQQ""8Y7777 * * *SVVS)) ) *!6NOOOOsP+I:DID  AI>FIF$$I?BI I1I,,I1rcddl}|dtd|} ||jd||}|jdkr3 td i|S#tdxYw| |d dS#t$r"}tt|d d}~wwxYw) a Retrieve the status of an extract job. .. deprecated:: The extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement. Args: job_id (str): The ID of the extract job. Returns: ExtractResponse[Any]: The status of the extract job. Raises: ValueError: If there is an error retrieving the status. rNr rr rrrzget extract statusrr1) rrrrrrWrrrBrrr^r0)r`rrr\rrs r#get_extract_statusz!V1FirecrawlApp.get_extract_statuss$   %     '')) *((DL)N)Nf)N)NPWXXH#s**T,??x}}???T#$RSSS""8-ABBBBB * * *SVVS)) ) *s)+B-#BBB-- C7CCc4ddl} | dtd|} |}|r||}|||||dt d} |r|| d<|r|| d <|r|| d < ||jd | | } | jd kr3 tdi| S#td xYw| | ddS#t$r"} tt| dd} ~ wwxYw)a Initiate an asynchronous extract job. .. deprecated:: The extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement. Args: urls (List[str]): URLs to extract information from prompt (Optional[str]): Custom extraction prompt schema (Optional[Any]): JSON schema/Pydantic model system_prompt (Optional[str]): System context allow_external_links (Optional[bool]): Follow external links enable_web_search (Optional[bool]): Enable web search show_sources (Optional[bool]): Include source URLs agent (Optional[Dict[str, Any]]): Agent configuration idempotency_key (Optional[str]): Unique key to prevent duplicate requests Returns: ExtractResponse[Any] with: * success (bool): Whether request succeeded * data (Optional[Any]): Extracted data matching schema * error (Optional[str]): Error message if any Raises: ValueError: If job initiation fails rNr rr rqrr*rrrrrz async extractrr1)rrrrrrrrWrrrBrrr^r0)r`rr*rerPrQrRrSrrr\rrrs r# async_extractzV1FirecrawlApp.async_extract;sN   %     ''))  6--f55F"60'-G--     ,%+L "  9+8L (  *$)L ! *))T\*F*F*F V]^^H#s**T,??x}}???T#$RSSS""8_===== * * *SVVS)) ) *s*6*C+!B??CC++ D5DDmax_urlsshow_full_textr6experimental_streamr$r%r6r&crt||||}||||||}|jr|jst ddddS|j} ||} | jd kr| S| jdkr| S| jd krt dd ddStjd b) a Generate LLMs.txt for a given URL and poll until completion. Args: url (str): Target URL to generate LLMs.txt from max_urls (Optional[int]): Maximum URLs to process (default: 10) show_full_text (Optional[bool]): Include full text in output (default: False) cache (Optional[bool]): Whether to use cached content if available (default: True) experimental_stream (Optional[bool]): Enable experimental streaming Returns: GenerateLLMsTextStatusResponse with: * Generated LLMs.txt content * Full version if requested * Generation status * Success/error information Raises: Exception: If generation fails r4r5r6__experimental_streamr#Fz#Failed to start LLMs.txt generationr^rrrrrTr]r/LLMs.txt generation job terminated unexpectedlyr) r2async_generate_llms_textrrrMcheck_generate_llms_text_statusrrr) r`rJr$r%r6r&paramsrrrs r#generate_llms_textz!V1FirecrawlApp.generate_llms_texts8*'"5    00 ) 3 1   x{ 3;   99&AAF} ++ (** ,..7!K#  JqMMM r2c`t||||}|}d|i|dd}dt|d< ||jd||} | } td|td | | d r! tdi| S#td xYw| | d n.#t$r!} tt| d } ~ wwxYwtddS)a Initiate an asynchronous LLMs.txt generation operation. Args: url (str): The target URL to generate LLMs.txt from. Must be a valid HTTP/HTTPS URL. max_urls (Optional[int]): Maximum URLs to process (default: 10) show_full_text (Optional[bool]): Include full text in output (default: False) cache (Optional[bool]): Whether to use cached content if available (default: True) experimental_stream (Optional[bool]): Enable experimental streaming Returns: GenerateLLMsTextResponse: A response containing: * success (bool): Whether the generation initiation was successful * id (str): The unique identifier for the generation job * error (str, optional): Error message if initiation failed Raises: Exception: If the generation job initiation fails. r(rJTrsrqr /v1/llmstxt json_datarrrzstart LLMs.txt generationNFInternal server errorrr1)r2rrzrrrWrBrrrGrrr^r0) r`rJr$r%r6r&r.r\r2reqrrs r#r,z'V1FirecrawlApp.async_generate_llms_textsp6*'"5    ''))CQ6;;4;#P#PQ 5G55 ( %$$ %A%A%A9gVVCxxzzH +y ) ) ) *h ' ' '||I&& JS5AAAAAS#$QRRR""8-HIIII % % %SVV$$ $ %*)    s+A(C1: CCC11 D;DDc |} ||jd||}|jdkrR |}t d i|S#t $r$}t dt|d}~wwxYw|jdkrt d||dn.#t $r!}tt|d}~wwxYwt dd d d S)a; Check the status of a LLMs.txt generation operation. Args: id (str): The unique identifier of the LLMs.txt generation job to check status for. Returns: GenerateLLMsTextStatusResponse: A response containing: * success (bool): Whether the generation was successful * status (str): Status of generation ("processing", "completed", "failed") * data (Dict[str, str], optional): Generated text with fields: * llmstxt (str): Generated LLMs.txt content * llmsfulltxt (str, optional): Full version if requested * error (str, optional): Error message if generation failed * expiresAt (str): When the generated data expires Raises: Exception: If the status check fails. /v1/llmstxt/rzFFailed to parse Firecrawl response as GenerateLLMsTextStatusResponse: Nz!LLMs.txt generation job not foundz check LLMs.txt generation statusFr3r^rr*r1) rrrWrrBrMrr0rr^)r`rr\rr2rs r#r-z.V1FirecrawlApp.check_generate_llms_text_status s>('')) %((DL)J)Jb)J)JGTTH#s**w ( I;HHiHHH www#$umpqrmsms$u$uvvvw%,, CDDD""8-OPPPP % % %SVV$$ $ %0E\emy{||||s5+CA"" B,B  B3C C/C**C/c<|rdd|j|dSdd|jdS)a$ Prepare the headers for API requests. Args: idempotency_key (Optional[str]): A unique key to ensure idempotency of requests. Returns: Dict[str, str]: The headers including content type, authorization, and optionally idempotency key. zapplication/jsonr) Content-Typerzx-idempotency-key)r9r)rV)r`rs r#rzV1FirecrawlApp._prepare_headers1 sO   2!94Q'\:;;;;r2ct|D]A}tj||}|jdkrt j|d|zz>|cS|S)a  Make a GET request with retries. Args: url (str): The URL to send the GET request to. headers (Dict[str, str]): The headers to include in the GET request. retries (int): Number of retries for the request. backoff_factor (float): Backoff factor for retries. Returns: requests.Response: The response from the GET request. Raises: requests.RequestException: If the request fails after the specified retries. r\r?r)r@rrrrrr`rJr\r<r=rArs r#rzV1FirecrawlApp._get_requesth se*W~~  G|C999H#s** >Q'\:;;;;r2ct|D]A}tj||}|jdkrt j|d|zz>|cS|S)a Make a DELETE request with retries. Args: url (str): The URL to send the DELETE request to. headers (Dict[str, str]): The headers to include in the DELETE request. retries (int): Number of retries for the request. backoff_factor (float): Backoff factor for retries. Returns: requests.Response: The response from the DELETE request. Raises: requests.RequestException: If the request fails after the specified retries. rCr?r)r@rdeleterrrrDs r#rzV1FirecrawlApp._delete_request se*W~~  GsG<<Q'\:;;;;r2c |jd|}|||}|jdkr: |}n#t dxYw|ddkrd|vr|d}d|vrt |dd krns||d|} |}n#t dxYw||dgd|v||d<tdi|St d |dd vr%t|d }tj |n.t d |d| |d)a Monitor the status of a crawl job until completion. Args: id (str): The ID of the crawl job. headers (Dict[str, str]): The headers to include in the status check requests. poll_interval (int): Seconds between status checks. Returns: CrawlStatusResponse: The crawl results if the job is completed successfully. Raises: Exception: If the job fails or an error occurs during status checks. Trrrrr]rrrz,Crawl job completed but no data was returnedactivepausedpendingqueuedwaitingrrz)Crawl job failed or was stopped. Status: rNr1) rWrrrBrrrrrmaxrrr)r`rr\rrWrrrs r#rz"V1FirecrawlApp._monitor_job_status s& J5555G"//AAO*c11T"1"6"6"8"8KKT#$RSSSx(K77,,*62$ 33";v#6771<< %.2.?.? F@SU\.].]O`.=.B.B.D.D `&/0^&_&_ _ KK (C(CDDD% 33/3 F+4CC{CCC'(VWWW *.nnn"%mA"6"6MJ}----#$gP[\dPe$g$ghhh""?4HIII? JsAA-CCrr}c |}|dd}|dd}nd# |jdd}|rd|}d|j}n d |j}d }n#t $rd |j}d }YnwxYwYnxYw||j|||}tj || ) ah Handle errors from API responses. Args: response (requests.Response): The response object from the API request. action (str): Description of the action that was being performed. Raises: Exception: An exception with a message containing the status code and error details from the response. rNo error message provided.details%No additional error details provided.Nrz#Server returned non-JSON response: zFull response status: z+Server returned empty response with status zNo additional details availablez0Server returned unreadable response with status )r) rBrrrrr^_get_error_messager exceptions HTTPError)r`rr}r error_message error_details response_textmessages r#rzV1FirecrawlApp._handle_error s& B$MMOOM)--g7STTM)--i9`aaMM B B ( dsd 3  &&((F$Y-$Y$YM$SX=Q$S$SMM$hRZRf$h$hM$EM B B B iS[Sg i i A  B))(*> Wdee!++Gh+GGGs0AAB$?BB$BB$BB$rrVrWc|dkr d|d|d|S|dkr d|d|d|S|dkr d|d |d|S|d kr d |d |d|S|d kr d|d|d|Sd|d|d|d|S)a Generate a standardized error message based on HTTP status code. Args: status_code (int): The HTTP status code from the response action (str): Description of the action that was being performed error_message (str): The error message from the API response error_details (str): Additional error details from the API response Returns: str: A formatted error message izPayment Required: Failed to z. z - iz!Website Not Supported: Failed to izRequest Timeout: Failed to z as the request timed out. izConflict: Failed to z due to a conflict. rz!Internal Server Error: Failed to zUnexpected error during z: Status code r1r`rr}rVrWs r#rSz!V1FirecrawlApp._get_error_message s #  ]&]]M]]m]] ] C  bvbbbbS`bb b C  uuuTauufsuu u C  g&ggmggXegg g C  bvbbbbS`bb btfttKttS`ttertt tr2)r time_limitr$analysis_promptrP*_V1FirecrawlApp__experimental_stream_steps on_activity on_sourcer\r]r^r_r`ci} ||| d<||| d<||| d<||| d<||| d<||| d<tdi| } |||||||} | d rd | vr| S| d } d } d } || }|r9d |vr5|d | d}|D] }||t |d } | r9d|vr5|d|d}|D] }| |t |d}|ddkr|S|ddkr%t d|d|ddkrnt jddddSa Initiates a deep research operation on a given query and polls until completion. Args: query (str): Research query or topic to investigate max_depth (Optional[int]): Maximum depth of research exploration time_limit (Optional[int]): Time limit in seconds for research max_urls (Optional[int]): Maximum number of URLs to process analysis_prompt (Optional[str]): Custom prompt for analysis system_prompt (Optional[str]): Custom system prompt __experimental_stream_steps (Optional[bool]): Enable experimental streaming on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth} on_source (Optional[Callable]): Source discovery callback receiving {url, title, description} Returns: DeepResearchStatusResponse containing: * success (bool): Whether research completed successfully * status (str): Current state (processing/completed/failed) * error (Optional[str]): Error message if failed * id (str): Unique identifier for the research job * data (Any): Research findings and analysis * sources (List[Dict]): List of discovered sources * activities (List[Dict]): Research progress log * summaries (List[str]): Generated research summaries Raises: Exception: If research fails Nrr;r4r=r__experimental_streamSteps)rr\r$r]rPrrrTrDrrr]r^zDeep research failed. Error: rrrFz)Deep research job terminated unexpectedlyrr1)r9async_deep_researchrcheck_deep_research_statusrrrr)r`r rr\r$r]rPr^r_r`research_paramsrrlast_activity_countlast_source_countrnew_activitiesactivity new_sourcessources r# deep_researchzV1FirecrawlApp.deep_research s6P  *3OJ '  !+5OK (  )1OI &  &0?O, -  $.;ON + & 2*>O$ 44V<&?&?# ;Y&00$Y/0A0B0BC )&&FIf%%%%$'y(9$:$:!h;.. !X-- U 7@S@S U UVVV!\11 JqMMM- 0!+VWWWr2)rr\r$r]rPr^ci}|||d<|||d<|||d<|||d<|||d<|||d<tdi|}|} d|i|d d } d t| d <d | vrG| d } | r=d| vr9t | ddr#| d| d d< ||jd| | } | jdkr) | S#tdxYw| | dn.#t$r!} tt| d} ~ wwxYwdddS) Initiates an asynchronous deep research operation. Args: query (str): Research query or topic to investigate max_depth (Optional[int]): Maximum depth of research exploration time_limit (Optional[int]): Time limit in seconds for research max_urls (Optional[int]): Maximum number of URLs to process analysis_prompt (Optional[str]): Custom prompt for analysis system_prompt (Optional[str]): Custom system prompt __experimental_stream_steps (Optional[bool]): Enable experimental streaming Returns: Dict[str, Any]: A response containing: * success (bool): Whether the research initiation was successful * id (str): The unique identifier for the research job * error (str, optional): Error message if initiation failed Raises: Exception: If the research initiation fails. Nrr;r4r=rrcr Trsrqrrre/v1/deep-researchrrzstart deep researchFr3rr1)r9rrzrhasattrrerrWrrBrrr^r0)r`r rr\r$r]rPr^rfr\r2 json_optsrrs r#rdz"V1FirecrawlApp.async_deep_researchu s>  *3OJ '  !+5OK (  )1OI &  &0?O, -  $.;ON + & 2x5H5O5O5Q5Q -(2 %))T\*L*L*LiY`aaH#s**S#==??*S#$QRRR""8-BCCCC % % %SVV$$ $ %!+BCCCs**D+,DDD++ E5EEc|} ||jd||}|jdkr) |S#t dxYw|jdkrt d||dn.#t $r!}tt|d}~wwxYwdd d S)  Check the status of a deep research operation. Args: id (str): The ID of the deep research operation. Returns: DeepResearchResponse containing: Status: * success - Whether research completed successfully * status - Current state (processing/completed/failed) * error - Error message if failed Results: * id - Unique identifier for the research job * data - Research findings and analysis * sources - List of discovered sources * activities - Research progress log * summaries - Generated research summaries Raises: Exception: If the status check fails. /v1/deep-research/rrr7zDeep research job not foundzcheck deep research statusNFr3r) rrrWrrBrrr^r0)r`rr\rrs r#rez)V1FirecrawlApp.check_deep_research_status s2'')) %((DL)P)PB)P)PRYZZH#s**S#==??*S#$QRRR%,, =>>>""8-IJJJJ % % %SVV$$ $ %!+BCCCs)+BAA''3B C%CCr method_namec &|sdShdhdhdhdhdhdhdhdd}||t}t||z }|r)td |d d |d dS) a Validate additional keyword arguments before they are passed to the API. This provides early validation before the Pydantic model validation. Args: kwargs (Dict[str, Any]): Additional keyword arguments to validate method_name (str): Name of the method these kwargs are for Raises: ValueError: If kwargs contain invalid or unsupported parameters N>rrvrRrOrnrlrtrurerhrrcrbrjrdrgrfrk> r rrrrXrtrurr>rrrrrrrrrrrrQrr>rrrtrrrr>rr*rerrSrPrRrQ>rrrvrRrOrnr\rtrrurerhrcrbrjrdrgrf)rprrrrOrrrzUnsupported parameter(s) for z: z, zC. Please refer to the API documentation for the correct parameters.)rsetkeysr^r)r`rrv method_paramsallowed_paramsunknown_paramss r#rzV1FirecrawlApp._validate_kwargs s7  FKKK|{{}||RRR"@"@"@(F(F(F,J,J,J%  2'**;>>V[[]]++n<  lk[kkDIIVdLeLekkkll l l lr2c||St|trHt|dr|St|dr|St|t r fd|DSt|ttfrfd|DS|S)zw Utility to ensure a schema is a dict, not a Pydantic model class. Recursively checks dicts and lists. Nmodel_json_schemarecBi|]\}}||Sr1r)r|kvr`s r# z6V1FirecrawlApp._ensure_schema_dict..& s-NNNtq!At//22NNNr2c:g|]}|Sr1r)r|rr`s r#r~z6V1FirecrawlApp._ensure_schema_dict..( s'@@@AD,,Q//@@@r2) ryrrqr~rerzitemslisttupler`res` r#rz"V1FirecrawlApp._ensure_schema_dict s >M fd # # 'v233 '//111** '}}& fd # # ONNNNv||~~NNN N ftUm , , A@@@@@@@ @ r2c |rt|ttfsdS|t}ddl}|j|dt }||vrdS|| t|tr d|vrt|dt r|dd}t|dkru|dd kri|d d kr]|d }||kr | |dS||vr3| |||||| |S| D]2} | | |||r| |dS3nJt|tr5|D]2} | | |||r| |dS3| |n#| |wxYwdS) a Check if an object contains a recursive reference to a specific definition. Args: obj: Object to check target_def_name: Name of the definition to check for recursion defs: Dictionary of definitions visited: Set of visited object keys to detect cycles Returns: True if recursive reference is found, False otherwise FNrT) sort_keysdefault$ref/r:#r$defs) ryrzrrxrBdumpsr0addsplitrdiscard_contains_recursive_refvalues) r`objtarget_def_namedefsvisitedrBobj_keyref_pathdef_namevalueitems r#rz&V1FirecrawlApp._contains_recursive_ref+ sT *S4,77 5 ?eeG $*SD#>>> g  5 G %#t$$ $S==ZF S%A%A="6{0055H8}}))hqkS.@.@Xa[T[E[E[#+B<#66#' OOG $ $ $ $ $$t++#'#?#?XP_aegn#o#o OOG $ $ $ $!ZZ\\$$E33E?DRYZZ$# OOG $ $ $ $ $$$C&& $$$D33D/4QXYY$# OOG $ $ $ $ $ $ OOG $ $ $ $GOOG $ $ $ $us%)B G) !G)/G)4G)G))Hcv|sdS|D]\}}||||rdS dS)z Check if $defs contain circular references. Args: defs: Dictionary of definitions to check Returns: True if circular references are found, False otherwise FT)rr)r`rr def_values r#_check_for_circular_defsz'V1FirecrawlApp._check_for_circular_defs] sZ 5#'::<<   Hi++IxFF tt ur2rc|r"t|ttfrdkr|Stt |}|vr|S| t|tr#d|vrt|dt r|dd}t|dkrg|ddkr[|dd krO|d }|vrC t|dz |S| |Si}| D]*\} } | d kr | dz|| <+| |St|tr&fd |D |S  |n# |wxYw|S) a| Resolve $ref references in a JSON schema object. Args: obj: Object to resolve references in defs: Dictionary of definitions visited: Set to track visited objects and prevent infinite recursion depth: Current recursion depth Returns: Object with resolved references r3Nrrr:rrrrrc Fg|]}|dzS)r) _resolve_refs)r|rrdepthr`rs r#r~z0V1FirecrawlApp._resolve_refs.. s1[[[t**4w JJ[[[r2) ryrzrrxrrr0rrrrr) r`rrrrobj_idrrresolvedrrs ` ``` r#rzV1FirecrawlApp._resolve_refsp sL *S4,77 52::J ?eeGC W  J F $#t$$ \S==ZF S%A%A="6{0055H8}}))hqkS.@.@Xa[T[E[E[#+B<#t++#'#5#5d4>6J6JDRY[`cd[d#e#e OOF # # # # OOF # # # #"%))++XXJCg~~ $($6$6udGUUVY$W$WHSMM OOF # # # # C&& \[[[[[[[WZ[[[ OOF # # # #  \ OOF # # # #GOOF # # # # s &B4G10G1AG1%G11Hcv|rt|ts|Stfd|S)z Normalize a schema for OpenAI compatibility by handling recursive references. Args: schema: Schema to normalize Returns: Normalized schema ct|ts%t|tr fd|DS|St|}| vr|S |t|dvr |Sdvrd}i}D]2\}}t|trd|vr |||<-|||<3i|d fd|Di} ||Sddkrdvrdd urd=ddkrad vr]dvrYtd tr;tdtr fd d D}|r|d <nd =nd =tD]S\}}t|trd|vr ||<-t|tr fd |D|<T |S) Nc&g|] }|Sr1r1r|rnormalize_objects r#r~zYV1FirecrawlApp._normalize_schema_for_openai..normalize_object.. s%CCCt,,T22CCCr2rrc.i|]\}}||Sr1r1)r|rrrs r#rzYV1FirecrawlApp._normalize_schema_for_openai..normalize_object.. s.5f5f5fQUQRTUa9I9I!9L9L5f5f5fr2robject propertiesadditionalPropertiesTrequiredc(g|]}|dv |S)rr1)r|field normalizeds r#r~zYV1FirecrawlApp._normalize_schema_for_openai..normalize_object.. s5&I&I&I&+z,/G&G&G',&G&G&Gr2cTg|]$}t|tr |n|%Sr1rxrs r#r~zYV1FirecrawlApp._normalize_schema_for_openai..normalize_object.. s<&t&t&tfjDRVAWAW'a'7'7'='='=]a&t&t&tr2) ryrzrrrrpoprr) rrrprocessed_restrrresultvalid_requiredrrrs @r#rzEV1FirecrawlApp._normalize_schema_for_openai..normalize_object sc4(( c4((DCCCCsCCCC WWF   KK   cJ##'''!!*$$!~~g..!#","2"2"4"444JC"5$//4F%4G4G.>.>u.E.Es++.3s++gNgG5f5f5f5fY]YcYcYeYe5f5f5fgg''' v&&(22 **566$>>56v&&(22j(( **z*5t<< /z,7>> /&I&I&I&IJ9O&I&I&IN%31? :..&z22":.#:#3#3#5#566 u u UeT**uvU/B/B&6&6u&=&=JsOOt,,u&t&t&t&tns&t&t&tJsO OOF # # # r2ryrzrx)r`rerrs @@r#_normalize_schema_for_openaiz+V1FirecrawlApp._normalize_schema_for_openai s_ Z55 M%%> > > > > > @ '''r2cx|rt|tsdStfd| S)z Validate schema for OpenAI compatibility. Args: schema: Schema to validate Returns: True if schema is valid, False otherwise Tct|tsdSt|}|vrdS|d|vr|dS|ddkr6d|vr2d|vr.|ddur|dS|D]}t|tr(d|vr$|r|dS?t|trB|D]?}t|tr(d|vr$|r|dS@|dS) NFrrrrpatternPropertiesrT)ryrzrrrrrr)rrrrhas_invalid_structurers r#rzIV1FirecrawlApp._validate_schema_for_openai..has_invalid_structure sc4(( uWWF  u KK   }}'''u8++C''#3...//477'''t , ,eT** ,vU/B/B,,U33$///#tt$ t,,, %,,%dD11,fD6H6H44T::, ' 7 7 7'+ttt OOF # # #5r2r)r`rerrs @@r#_validate_schema_for_openaiz*V1FirecrawlApp._validate_schema_for_openai sb Z55 4%%# # # # # # J)(0000r2c|rt|tsdSddl}|j|}d|vpd|vpd|v}t |dp|d}|p|S) z Detect if a schema contains recursive references. Args: schema: Schema to analyze Returns: True if schema has recursive patterns, False otherwise FrN"$ref"#/$defs/z#/definitions/r definitions)ryrzrBrrr)r`rerB schema_stringhas_refshas_defss r#_detect_recursive_schemaz'V1FirecrawlApp._detect_recursive_schema) s Z55 5 " 6**  % . - ' .  -   7++Hvzz-/H/HII#8#r2cN|sdddS||rdddSdddS)z Select appropriate model based on schema complexity. Args: schema: Schema to analyze Returns: Dict with modelName and reason z gpt-4o-mini no_schema) modelNamereasonzgpt-4orecursive_schema_detected simple_schema)rrs r#_select_model_for_schemaz'V1FirecrawlApp._select_model_for_schemaA sO G!.+FF F  ( ( 0 0 R!)5PQQ Q*oFFFr2ct|trh||vrc|||}|rt|tr|di}ddl}|j|}|dpd|vpd|v}|r` |||}|j|} d| vpd| v} | s|}t|trd|vr|d=nT#t$rYnHwxYw |||}t|trd|vr|d=n#t$rYnwxYw||}| |st|j |||<|S)a1 Process and validate a schema container (like extract or json_options). Args: schema_container: Dict containing schema schema_key: Key where schema is stored (default: "schema") Returns: Processed schema container rrNrr) ryrzrrrBrrrrrr^OPENAI_SCHEMA_ERROR_MESSAGE) r`schema_container schema_keyrerrBr has_any_refsresolved_schemaresolved_stringhas_remaining_refss r#_process_schema_with_validationz.V1FirecrawlApp._process_schema_with_validationS s & - -* 2*@P2P2P--.>z.JKKF *VT22 zz'2.. * 6 2 2 JJw''0 -0-/   *.*<*>F33F;; C !ABBB+1 Z (s%AC(( C54C592D,, D98D9NN)FNr:r;)Nr)re)^r+r,r-r.rrr0rarr rrrrWrrrrrrrrrrrrcrrrprirrrrr"rr*rr0rr[rrrrrrrrrrrr'rrrrrrrrrOr r"rMr/rGr,r-rrrResponserrrrrrSrrBrmrdrerrrrrrrrrrr1r2r#rUrUs$ q QQ Qx}QX\QQQQ,mq04040404&*%*37%)4837(,OS(,.237EIIM%)-1261U7U7U7U7d7,g$hij U7 d38n- U7 #49- U7#49-U7 (~U7smU7c]U7/0U7TNU7$,D>U7#+4.U7 ~U7 G$JKL!U7" ~#U7$l+%U7&#<0'U7(d57I=ZgivyGIWYrt*$@AB)U7*&..E%F+U7,c]-U7.%TN/U70"*$1U72*#.3U7U7U7U7v$(!%$("&%)&*%*8<]3]3]3]3C= ]3 # ]3 SM ]33-]3c]]3sm]3c]]3%_5]3*]3]3]3]3~="7====,#@<$}%@<&#4.'@<("#)@<*&d^+@<, }-@<."#/@<2 3@<@<@<@#z<$}%z<&#4.'z<("#)z<*&d^+z<,"#-z<0 1z<z<z<zKC3-KC#KC+,KCKC (~KC'tnKCD>KC  FGH!KC",'#KC$|,%KC&$u\3E}VceruCESUnp{&{ |}~'KC(')KC* }+KC,"#-KC.&d^/KC0"#1KC4 %5KCKCKCKCbW[,0,0,0,0"&!&/3!%04/3$(KO*./3AE*.)-)-.2/JCJCJC3iJC$w(Q RST JC $sCx.) JC tCy) JCtCy)JC$D>JC3-JC#JC+,JCJC (~JC'tnJCD>JC  FGH!JC",'#JC$|,%JC&$u\3E}VceruCESUnp{&{ |}~'JC(')JC*"#+JC,"#-JC.&d^/JC2 3JCJCJCJC`W[,0,0,0,0"&!&/3!%04/3$(KO*./3AE*.)-.2)-/JCJCJC3iJC$w(Q RST JC $sCx.) JC tCy) JCtCy)JC$D>JC3-JC#JC+,JCJC (~JC'tnJCD>JC  FGH!JC",'#JC$|,%JC&$u\3E}VceruCESUnp{&{ |}~'JC(')JC*"#+JC,&d^-JC."#/JC2 3JCJCJCJCX<FC<F4O<F<F<F<F|FCF4IFFFF>)-~P%)$(+/3805+0.2~P~P~P49%~PSM ~P SM ~P $C= ~P#+4.~P (~~P#4.~PDcN+~P+3/~P~P~P~P@%*%*1B31G%*%*%*%*R)-O*%)$(+/3805+0.2O*O*O*49%O*SM O* SM O* $C= O*#+4.O* (~O*#4.O*DcN+O*8I7MO*O*O*O*j'+-1$(26CCCCsm C %TN C D> C"*$C<\CCCCR'+-1$(268 8 8 8 sm 8 %TN 8 D> 8 "*$8 FJDH^X^X^X^X } ^X ! ^X sm ^X&c]^X$C=^X*2$^X"(DcN+;T+A"BC^X $sCx.)94)? @A^XNj^X^X^X^XH(,(,&*-1+/:>EDEDEDED } ED ! ED sm ED&c]ED$C=ED*2$EDDHS>EDEDEDEDN(DS(D5Q(D(D(D(DT.ltCH~.lC.lD.l.l.l.lf$0000d&0000dO(O(O(b414141l$$$0GGGG$6 6 6 6 6 6 r2rUceZdZdZdedefdZddZddZd ed e e ee fgdfddfd Z d ed e ee fddfd Z de ee fddfdZdS)ra A class to watch and handle crawl job events via WebSocket connection. Attributes: id (str): The ID of the crawl job to watch app (V1FirecrawlApp): The V1FirecrawlApp instance data (List[Dict[str, Any]]): List of crawled documents/data status (str): Current status of the crawl job ws_url (str): WebSocket URL for the crawl job event_handlers (dict): Dictionary of event type to list of handler functions rappc||_||_g|_d|_|jddd||_gggd|_dS)Nrhttpwsr)donerdocument)rrrrrWreplacews_urlevent_handlers)r`rrs r#razV1CrawlWatcher.__init__ sc*,   ,,VT::JJbJJ   r2rXNcKtj|jddd|jjfg4d{V }||d{Vdddd{VdS#1d{VswxYwYdS)zU Establishes WebSocket connection and starts listening for messages. Nrr)max_sizeadditional_headers websocketsconnectrrrV_listenr` websockets r#rzV1CrawlWatcher.connect s(% K!02NDHeedeede.f*d?Z/ddddddddddddddddd7dd@d edAeeedBeeedCee dDee dEee dFeedGeedHeedIeedJee0dKeeee1fdLeedMeedNeedOee dPeed;ee deede2f(dQZ3ddddddddddddddddd7dd@d edAeeedBeeedCee dDee dEee dFeedGeedHeedIeedJee0dKeeee1fdLeedMeedNeedOee dPeed;ee deede4f(dRZ5dSede2fdTZ6ddSed eeefd;e de2fdUZ7dddddd"dddVd edWeedIeedXeedYeedEee d*ee d+eedee8de9fdZZ: ddddd[d[d[dd\d eeed]eed^eed_eedHeed`eedaeed:eeeefde;efdbZfddZ?dSede>fdeZ@dSedeeeffdfZAdgede;efdhZB ddddd[d[d[dd\d eeed]eed^eed_eedHeed`eedaeed:eeeefde;efdiZCddddjd edkee dleedmeedeDf dnZEdddddod edkee dleedpeedmeedeFf dqZGdSedeDfdrZHdddddddddsdtedCee duee dkee dveed_eedweedxeeIeeefgdfdyeeIeeefgdfdeJfdzZKddddddd{dtedCee duee dkee dveed_eedweedeeeffd|ZLdSedeJfd}ZMddddddd"ddd~ dtedEee deedeedeedeed+eed*ee dJee0deeeeefeNfdeOfdZPxZQS)AsyncV1FirecrawlAppz Asynchronous version of V1FirecrawlApp that implements async methods using aiohttp. Provides non-blocking alternatives to all V1FirecrawlApp operations. r\rVrWcNt||dS)N)rVrWsuperra)r`rVrW __class__s r#razAsyncV1FirecrawlApp.__init__ s& ':::::r2Nr:r;methodrJr\rr<r=rXc Ktj4d{V}t|D]&} |||||4d{V } | jdkr3t j|d|zzd{V dddd{Vc| jdkr || d|dd{V| d{Vcdddd{Vccdddd{VS#1d{VswxYwY#tj $r6} ||dz kr| t j|d|zzd{VYd} ~  d} ~ wwxYwtd #1d{VswxYwYdS) a Generic async request method with exponential backoff retry logic. Args: method (str): The HTTP method to use (e.g., "GET" or "POST"). url (str): The URL to send the request to. headers (Dict[str, str]): Headers to include in the request. data (Optional[Dict[str, Any]]): The JSON data to include in the request body (only for POST requests). retries (int): Maximum number of retry attempts (default: 3). backoff_factor (float): Factor to calculate delay between retries (default: 0.5). Delay will be backoff_factor * (2 ** retry_count). Returns: Dict[str, Any]: The parsed JSON response from the server. Raises: aiohttp.ClientError: If the request fails after all retries. Exception: If max retries are exceeded or other errors occur. N)rrJr\rBr?ri,zmake z requestrzMax retries exceeded) aiohttp ClientSessionr@requestrasynciorrrB ClientErrorr) r`rrJr\rr<r=sessionrArrs r#_async_requestz"AsyncV1FirecrawlApp._async_request s56(** 4 4 4 4 4 4 4g >> I I I&%3d /  55555555!#?c11")-!w,0O"P"PPPPPPPP$ 5555555555555 $?c11"&"4"4X?Wv?W?W?W"X"XXXXXXXX%-]]__444444555555555555555 4 4 4 4 4 4 4 4 4 4 4 4 4 4555555555555555*III'A+--!-!w,(GHHHHHHHHHHHHHHI233 3! 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4swE#D -C:=D E#AC:D %E#: D D D D  E# E+E E# EE## E-0E-cFK|d|||||d{VS)a Make an async POST request with exponential backoff retry logic. Args: url (str): The URL to send the POST request to. data (Dict[str, Any]): The JSON data to include in the request body. headers (Dict[str, str]): Headers to include in the request. retries (int): Maximum number of retry attempts (default: 3). backoff_factor (float): Factor to calculate delay between retries (default: 0.5). Delay will be backoff_factor * (2 ** retry_count). Returns: Dict[str, Any]: The parsed JSON response from the server. Raises: aiohttp.ClientError: If the request fails after all retries. Exception: If max retries are exceeded or other errors occur. POSTNr)r`rJrr\r<r=s r#_async_post_requestz'AsyncV1FirecrawlApp._async_post_request s8*((gtWn]]]]]]]]]r2cFK|d||d||d{VS)a Make an async GET request with exponential backoff retry logic. Args: url (str): The URL to send the GET request to. headers (Dict[str, str]): Headers to include in the request. retries (int): Maximum number of retry attempts (default: 3). backoff_factor (float): Factor to calculate delay between retries (default: 0.5). Delay will be backoff_factor * (2 ** retry_count). Returns: Dict[str, Any]: The parsed JSON response from the server. Raises: aiohttp.ClientError: If the request fails after all retries. Exception: If max retries are exceeded or other errors occur. GETNr)r`rJr\r<r=s r#_async_get_requestz&AsyncV1FirecrawlApp._async_get_request5 s8(((WdG^\\\\\\\\\r2rr}cFK |d{V}|dd}|dd}n!#tjd|jxYw||j|||d{V}tj|)aR Handle errors from async API responses with detailed error messages. Args: response (aiohttp.ClientResponse): The response object from the failed request action (str): Description of the action that was being attempted Raises: aiohttp.ClientError: With a detailed error message based on the response status: - 402: Payment Required - 408: Request Timeout - 409: Conflict - 500: Internal Server Error - Other: Unexpected error with status code NrrPrQrR?Failed to parse Firecrawl error response as JSON. Status code: )rBrrrr_get_async_error_messager`rr} error_datarVrWrYs r#rz!AsyncV1FirecrawlApp._handle_errorK s  {'}}......J&NN74PQQM&NN96]^^MM {%&yhphw&y&yzz z55hov}^kllllllll!'*** AA A)rrVrWc6K|||||Sa Generate a standardized error message based on HTTP status code for async operations. Args: status_code (int): The HTTP status code from the response action (str): Description of the action that was being performed error_message (str): The error message from the API response error_details (str): Additional error details from the API response Returns: str: A formatted error message rSr[s r#r#z,AsyncV1FirecrawlApp._get_async_error_messagef "&&{FM=YYYr2cK|}||jd|d{V}tdi|S)z8Get current credit usage and billing period (v1, async).rNr1)rr rWrr`r\resps r#rz$AsyncV1FirecrawlApp.get_credit_usageu sq'')),,| 2 2 2         %,,t,,,r2cK|}||jd|d{V}tdi|S)z7Get current token usage and billing period (v1, async).rNr1)rr rWr"r,s r#rz#AsyncV1FirecrawlApp.get_token_usage~ sq'')),,| 1 1 1         $++d+++r2r.rAsyncV1CrawlWatchercK||||d{V}|drd|vrt|d|Std)a Initiate an async crawl job and return an AsyncV1CrawlWatcher to monitor progress via WebSocket. Args: url (str): Target URL to start crawling from params (Optional[V1CrawlParams]): See V1CrawlParams model for configuration: URL Discovery: * includePaths - Patterns of URLs to include * excludePaths - Patterns of URLs to exclude * maxDepth - Maximum crawl depth * maxDiscoveryDepth - Maximum depth for finding new URLs * limit - Maximum pages to crawl Link Following: * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead * crawlEntireDomain - Follow parent directory links * allowExternalLinks - Follow external domain links * ignoreSitemap - Skip sitemap.xml processing Advanced: * scrapeOptions - Page scraping configuration * webhook - Notification webhook settings * deduplicateSimilarURLs - Remove similar URLs * ignoreQueryParameters - Ignore URL parameters * regexOnFullURL - Apply regex to full URLs idempotency_key (Optional[str]): Unique key to prevent duplicate requests Returns: AsyncV1CrawlWatcher: An instance to monitor the crawl job via WebSocket Raises: Exception: If crawl job fails to start Nrrr)rrr/r)r`rJr.rrs r#rz'AsyncV1FirecrawlApp.crawl_url_and_watch svL $33CQQQQQQQQ   i ( ( 9T^-C-C&~d';TBB B788 8r2rcK||||d{V}|drd|vrt|d|Std)a Initiate an async batch scrape job and return an AsyncV1CrawlWatcher to monitor progress. Args: urls (List[str]): List of URLs to scrape params (Optional[V1ScrapeParams]): See V1ScrapeParams model for configuration: Content Options: * formats - Content formats to retrieve * includeTags - HTML tags to include * excludeTags - HTML tags to exclude * onlyMainContent - Extract main content only Request Options: * headers - Custom HTTP headers * timeout - Request timeout (ms) * mobile - Use mobile user agent * proxy - Proxy type Extraction Options: * extract - Content extraction config * jsonOptions - JSON extraction config * actions - Actions to perform idempotency_key (Optional[str]): Unique key to prevent duplicate requests Returns: AsyncV1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket Raises: Exception: If batch scrape job fails to start Nrrr)rrr/r)r`rr.rbatch_responses r#rz/AsyncV1FirecrawlApp.batch_scrape_urls_and_watch swH $;;D&/ZZZZZZZZ   i ( ( @T^-C-C&~d';TBB B>?? ?r2rs)rnr\rbrcrdrertrurvrfrgrhrrirOrjrRrnrkrbrcrdrertrurvrfrgrhrrzrirOrjrRcK||d|}|dtd}|r||d<|r||d<|r||d<|r||d<|||d <|r||d <|r||d <| r| d d |d<| | |d<| | |d<| | |d<| | |d<|r||d<|||d<|}||}t |tr"d|vr||d|d<t |tr|n|d d |d<|}||}t |tr"d|vr||d|d<t |tr|n|d d |d<|rd|D|d<d|vr<|dr4d|dvr*||dd|dd<d|vr<|dr4d|dvr*||dd|dd<d}||j|||d{V}|drd|vrtdi|dSd|vrtd|d|dt|}td|) aA Scrape a single URL asynchronously. Args: url (str): Target URL to scrape formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc) headers (Optional[Dict[str, str]]): Custom HTTP headers include_tags (Optional[List[str]]): HTML tags to include exclude_tags (Optional[List[str]]): HTML tags to exclude only_main_content (Optional[bool]): Extract main content only wait_for (Optional[int]): Wait for a specific element to appear timeout (Optional[int]): Request timeout (ms) location (Optional[V1LocationConfig]): Location configuration mobile (Optional[bool]): Use mobile user agent skip_tls_verification (Optional[bool]): Skip TLS verification remove_base64_images (Optional[bool]): Remove base64 images block_ads (Optional[bool]): Block ads proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth) extract (Optional[V1JsonConfig]): Content extraction settings json_options (Optional[V1JsonConfig]): JSON extraction settings actions (Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]]): Actions to perform **kwargs: Additional parameters to pass to the API Returns: V1ScrapeResponse with: * success - Whether scrape was successful * markdown - Markdown content if requested * html - HTML content if requested * rawHtml - Raw HTML content if requested * links - Extracted links if requested * screenshot - Screenshot if requested * extract - Extracted data if requested * json - JSON data if requested * error - Error message if scrape failed Raises: Exception: If scraping fails rprqrrrnr\rorpNrqrrrtTrsrurvrwrxryrrrerOrcjg|]0}t|tr|n|dd1Srwrxr{s r#r~z2AsyncV1FirecrawlApp.scrape_url..Lrr2rRrrrrrr1) rrrrzrryrrWrrrr0)r`rJrnr\rbrcrdrertrurvrfrgrhrrirOrjrRrrrrr error_contents r#rpzAsyncV1FirecrawlApp.scrape_url s%z fl333((**-G--   /'.M) $  /'.M) $  8+7M- (  8+7M- (  (/@M+ ,  0'/M) $  /'.M) $  X(0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  +%*M' "  (1M* %  ..w77G'4(( PX-@-@$($<$.s*'l'l'lZ` TPT (U(U'l'l'lr2rRrrrqrrrrrrr1)rrzrryrrrrrrWrr_async_monitor_job_statusr)r`rrnr\rbrcrdrertrurvrfrgrhrrOrjrRrrrrrrrrrs r#rz%AsyncV1FirecrawlApp.batch_scrape_urlscsNz f&9:::   '.M) $  '.M) $  #+7M- (  #+7M- (  (/@M+ ,  '/M) $  '.M) $  (0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  %*M' "  ..w77G'4(( PX-@-@$($<$ #8{[dOeCeCe/3/G/G T]H^_gHh/i/iK "8 , K ' 'K ,F '8WbcpWqKqKq373K3KKXeLfgoLp3q3qK &x 0''8811| - - -           << " " C P\\$'' P NOOO77G]SSSSSSSS S   x)A B B B B Bs K&&K7)rnr\rbrcrdrertrurvrfrgrhrrOrjrRrrnrrncVK||di}|||d<|||d<|||d<|||d<|||d<|||d<|||d <| | d d |d <| | |d <| | |d<| | |d<| | |d<|||d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|}||}t|tr"d|vr||d|d<t|tr|n|d d |d<|rd|D|d<||d d |d<|||d<||t d!i|}|d d }||d<dt |d<d|vr<|dr4d|dvr*||dd|dd<d|vr<|dr4d|dvr*||dd|dd<||}||j d||d{V}| ddkr3 td!i| S#tdxYw||d d{VdS)"a, Initiate a batch scrape job asynchronously. Args: urls (List[str]): URLs to scrape formats (Optional[List[Literal]]): Content formats to retrieve headers (Optional[Dict[str, str]]): Custom HTTP headers include_tags (Optional[List[str]]): HTML tags to include exclude_tags (Optional[List[str]]): HTML tags to exclude only_main_content (Optional[bool]): Extract main content only wait_for (Optional[int]): Wait time in milliseconds timeout (Optional[int]): Request timeout in milliseconds location (Optional[LocationConfig]): Location configuration mobile (Optional[bool]): Use mobile user agent skip_tls_verification (Optional[bool]): Skip TLS verification remove_base64_images (Optional[bool]): Remove base64 encoded images block_ads (Optional[bool]): Block advertisements proxy (Optional[Literal]): Proxy type to use extract (Optional[JsonConfig]): Content extraction config json_options (Optional[JsonConfig]): JSON extraction config actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: V1BatchScrapeResponse with: * success - Whether job started successfully * id - Unique identifier for the job * url - Status check URL * error - Error message if start failed Raises: Exception: If job initiation fails rNrnr\rorprqrrrtTrsrurvrwrxryrrerOrcjg|]0}t|tr|n|dd1Srwrxr{s r#r~z?AsyncV1FirecrawlApp.async_batch_scrape_urls..Vrr2rRrrrrqrrrrrrr1)rrzrryrrrrrrWrrrBrr)r`rrnr\rbrcrdrertrurvrfrgrhrrOrjrRrrnrrrrrrs r#rz+AsyncV1FirecrawlApp.async_batch_scrape_urlssh| f&?@@@   '.M) $  '.M) $  #+7M- (  #+7M- (  (/@M+ ,  '/M) $  '.M) $  (0 tRV (W(WM* %  &,M( # ,3HM/ 0  +2FM. /  (1M* %  %*M' "  ..w77G'4(( PX-@-@$($<$ #8{[dOeCeCe/3/G/G T]H^_gHh/i/iK "8 , K ' 'K ,F '8WbcpWqKqKq373K3KKXeLfgoLp3q3qK &x 0''8811| - - -           << & &# - - P,??x}}??? P NOOO$$X/GHH H H H H H H H H Hs K99L )rrrrrrrrQrrrrrrrrrrrrrrrrrrQrrrrrrrrc@K||di}|||d<|||d<|||d<|||d<|||d<|||d<n|||d <| | |d <| | |d <| | d d |d<| | |d<| | |d<|||d<|||d<|||d<|||d<||tdi|}|d d }||d<dt|d<||}||jd||d{V}|drH |d}n#tdxYw| |||d{VS| |dd{VdS)a Crawl a website starting from a URL. Args: url (str): Target URL to start crawling from include_paths (Optional[List[str]]): Patterns of URLs to include exclude_paths (Optional[List[str]]): Patterns of URLs to exclude max_depth (Optional[int]): Maximum crawl depth max_discovery_depth (Optional[int]): Maximum depth for finding new URLs limit (Optional[int]): Maximum pages to crawl allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead crawl_entire_domain (Optional[bool]): Follow parent directory links allow_external_links (Optional[bool]): Follow external domain links ignore_sitemap (Optional[bool]): Skip sitemap.xml processing scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings deduplicate_similar_urls (Optional[bool]): Remove similar URLs ignore_query_parameters (Optional[bool]): Ignore URL parameters regex_on_full_url (Optional[bool]): Apply regex to full URLs delay (Optional[int]): Delay in seconds between scrapes allow_subdomains (Optional[bool]): Follow subdomains poll_interval (Optional[int]): Seconds between status checks (default: 2) idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: V1CrawlStatusResponse with: * Crawling status and progress * Crawled page contents * Success/error information Raises: Exception: If crawl fails rNrrrrrrrrrTrsrrrrrrrrJrqrrrrrrr1) rrzrrrrrrWrrr8r)r`rJrrrrrrrrQrrrrrrrrrrrrrrr\rrs r#rzAsyncV1FirecrawlApp.crawl_urlzsv fk222   $+8L (  $+8L (  '0L $  *0CL, -  $)L !  *0CL, - - ! -1EL- .  +1EL- .  %,:L )  %,:,?,?\`,?,a,aL )  &-L # # /5ML1 2 " .4KL0 1  (->L) *  $)L !  '.>L* + F###%44|44 "''D'II  E 7g 7 7 H''8811 \ $ $ $k7<<<<<<<< << " " B P\\$'' P NOOO77G]SSSSSSSS S$$X/@AA A A A A A A A A As ;EE"cKi}|||d<|||d<|||d<|||d<|||d<|||d<n|||d<| | |d <| | |d <| | d d |d <| | |d<| | |d<|||d<|||d<|||d<|||d<||tdi|}|d d }||d<dt|d<||}||jd||d{V}|dr! tdi|S#tdxYw| |dd{VdS)a Start an asynchronous crawl job. Args: url (str): Target URL to start crawling from include_paths (Optional[List[str]]): Patterns of URLs to include exclude_paths (Optional[List[str]]): Patterns of URLs to exclude max_depth (Optional[int]): Maximum crawl depth max_discovery_depth (Optional[int]): Maximum depth for finding new URLs limit (Optional[int]): Maximum pages to crawl allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead crawl_entire_domain (Optional[bool]): Follow parent directory links allow_external_links (Optional[bool]): Follow external domain links ignore_sitemap (Optional[bool]): Skip sitemap.xml processing scrape_options (Optional[ScrapeOptions]): Page scraping configuration webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings deduplicate_similar_urls (Optional[bool]): Remove similar URLs ignore_query_parameters (Optional[bool]): Ignore URL parameters regex_on_full_url (Optional[bool]): Apply regex to full URLs idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API Returns: V1CrawlResponse with: * success - Whether crawl started successfully * id - Unique identifier for the crawl job * url - Status check URL for the crawl * error - Error message if start failed Raises: Exception: If crawl initiation fails NrrrrrrrrrTrsrrrrrrrrJrqrrrrrr1) rzrrrrrrWrrrr)r`rJrrrrrrrrQrrrrrrrrrrrrrrr\rs r#rz#AsyncV1FirecrawlApp.async_crawl_urlscp   $+8L (  $+8L (  '0L $  *0CL, -  $)L !  *0CL, - - ! -1EL- .  +1EL- .  %,:L )  %,:,?,?\`,?,a,aL )  &-L # # /5ML1 2 " .4KL0 1  (->L) *  $)L !  '.>L* + F###%44|44 "''D'II  E 7g 7 7 H''8811 \ $ $ $           << " " B P&22222 P NOOO$$X/@AA A A A A A A A A As % D11Erc K|}d|}||j||d{V}|ddkrd|vr|d}d|vrt |ddkrn}|d}|st dnK|||d{V}||dg|}d|v||d<t|d|d |d|d |d |dd |vrd nd}d |vr|d |_ d|vr|d|_ |S)a7 Check the status and results of an asynchronous crawl job. Args: id (str): Unique identifier for the crawl job Returns: V1CrawlStatusResponse containing: Status Information: * status - Current state (scraping/completed/failed/cancelled) * completed - Number of pages crawled * total - Total pages to crawl * creditsUsed - API credits consumed * expiresAt - Data expiration timestamp Results: * data - List of crawled documents * next - URL for next page of results (if paginated) * success - Whether status check succeeded * error - Error message if failed Raises: Exception: If status check fails rNrr]rrrrrrrrFT)rrr]rrrr) rr rWrrrrrrrr r`rr\rrrrrrs r#rz&AsyncV1FirecrawlApp.check_crawl_statusgs2''))$$$ 33| 'X ' '         ??8 $ $ 3 3$$"6* ++;v.//144*v66H#'HIII&*&=&=h&P&P P P P P P PIKK fb 9 9:::"+K ++'+ F#(??8,,//'**!ook22# 66!ook22(($ 33EE    k ! !(__W55HN [ 'OOF33HMr2cK ||jd||d{V}|ddkrd|vr|d}d|vrt|ddkrn}|d}|std nK|||d{V}||dg|}d|v||d<tdi|Std |dd vr)tj t|d d{Vntd |d`)a Monitor the status of an asynchronous job until completion. Args: id (str): The ID of the job to monitor headers (Dict[str, str]): Headers to include in status check requests poll_interval (int): Seconds between status checks (default: 2) Returns: V1CrawlStatusResponse: The job results if completed successfully Raises: Exception: If the job fails or an error occurs during status checks TrNrr]rrrrz&Job completed but no data was returnedrHrz#Job failed or was stopped. Status: r1) r rWrrrrrrrrrrN)r`rr\rrrrrs r#r8z-AsyncV1FirecrawlApp._async_monitor_job_statuss _ $ 7 7<//2//!!K x((K77[((&v.D K//{6233q88!#.??6#:#:'""NN+LMMM!*.*A*A(G*T*T$T$T$T$T$T$T  IMM&"$=$=>>>&/ !K//+/K'0??;???#$LMMM**.nnnmC q$9$9:::::::::: ]kRZF[ ] ]^^^5 _r2)rrrrrrtrur.rrrcdKi} | r*| | dd||| d<||| d<||| d<||| d<||| d<||| d <||dd| d <tdi| } | dd} || d <d t| d <d} ||j| | dd|jid{V}|drd|vr tdi|Sd|vrtd|dtd|)a Asynchronously map and discover links from a URL. Args: url (str): Target URL to map params (Optional[V1MapParams]): See V1MapParams model: Discovery Options: * search - Filter pattern for URLs * ignoreSitemap - Skip sitemap.xml * includeSubdomains - Include subdomain links * sitemapOnly - Only use sitemap.xml Limits: * limit - Max URLs to return * timeout - Request timeout (ms) Returns: V1MapResponse with: * Discovered URLs * Success/error status Raises: Exception: If mapping fails TrsNrrrrrrtrurJrqrrrrrCrrNrzFailed to map URL. Error: r1) rrzrrrrWrVrrr)r`rJrrrrrrtrur.rrrrrs r#rzAsyncV1FirecrawlApp.map_urlsH  M   fkk4dkKK L L L  #)Jx  %*8J '  ).@J* +  #(4J} %  "'Jw   $+Jy !  %-]]Dt]%T%TJz "#00Z00 "''D'II  E 7g 7 7 H11| 'X ' ' $&> &>&>?2         << " " Ew(':': ,,8,, ,  L'9JLLMM MCCCDD Dr2Frr*rerPrRrSc\Kddl} | dtd|} |s|st d|s|st d|r||}|pg||||dt d } |r|| d <|r|| d <|r|| d <||jd | | d{V} | dr| d} | std | |jd| | d{V}|ddkr tdi|S|ddvr!td|dd|dtjdd{Vtd| d)aQ Asynchronously extract structured information from URLs. .. deprecated:: The extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement. Args: urls (Optional[List[str]]): URLs to extract from prompt (Optional[str]): Custom extraction prompt schema (Optional[Any]): JSON schema/Pydantic model system_prompt (Optional[str]): System context allow_external_links (Optional[bool]): Follow external links enable_web_search (Optional[bool]): Enable web search show_sources (Optional[bool]): Include source URLs agent (Optional[Dict[str, Any]]): Agent configuration Returns: V1ExtractResponse with: * Structured data matching schema * Source information if requested * Success/error status Raises: ValueError: If prompt/schema missing or extraction fails rNr rr r r rqrr*rrrrrrTrrr]rrrrrr1)rrrrr^rr$rrWrrr rrr)r`rr*rerPrQrRrSrrr\rrrrs r#rOzAsyncV1FirecrawlApp.extract!sN   %     '')) Df DBCC C BF B@AA A  6--f55FJB"60'3KMM33     ,%+L "  9+8L (  *$)L !11| ( ( (           << " " R\\$''F M KLLL '$($;$;|9999%% x(K77,;;{;;; *.EEE#$i;x3H$i$iS^_fSg$i$ijjjmA&&&&&&&&& 'Pg9N9NPPQQ Qr2c TK|}d|}||j||d{V}|ddkrd|vr|d}d|vrt|ddkrn}|d}|st dnK|||d{V}||dg|}d|v||d<t|d|d |d|d |d |d }d |vr |d |d <d|vr |d|d<dd |vrdndi|S)a2 Check the status of an asynchronous batch scrape job. Args: id (str): The ID of the batch scrape job Returns: V1BatchScrapeStatusResponse containing: Status Information: * status - Current state (scraping/completed/failed/cancelled) * completed - Number of URLs scraped * total - Total URLs to scrape * creditsUsed - API credits consumed * expiresAt - Data expiration timestamp Results: * data - List of scraped documents * next - URL for next page of results (if paginated) * success - Whether status check succeeded * error - Error message if failed Raises: Exception: If status check fails rNrr]rrrrrrrrrrFT) rr rWrrrrrrr>s r#rz-AsyncV1FirecrawlApp.check_batch_scrape_statuss2''))+r++ 33| 'X ' '         x K / /$$"6* ++;v.//144*v66H#'HIII&*&=&=h&P&P P P P P P PIKK fb 9 9:::"+K ++'+ F#.??8,,//'**!ook22# 66!ook22((     k ! ! +G 4HW  [ *62HV  ; 6 6uuD   r2c|K|}||jd|d|d{VS)a] Get information about errors from an asynchronous batch scrape job. Args: id (str): The ID of the batch scrape job Returns: V1CrawlErrorsResponse containing: errors (List[Dict[str, str]]): List of errors with fields: * id (str): Error ID * timestamp (str): When the error occurred * url (str): URL that caused the error * error (str): Error message * robotsBlocked (List[str]): List of URLs blocked by robots.txt Raises: Exception: If error check fails rrNrr rWr`rr\s r#rz-AsyncV1FirecrawlApp.check_batch_scrape_errorssh&'')),,| 9 9b 9 9 9          r2c|K|}||jd|d|d{VS)aa Get information about errors from an asynchronous crawl job. Args: id (str): The ID of the crawl job Returns: V1CrawlErrorsResponse containing: * errors (List[Dict[str, str]]): List of errors with fields: - id (str): Error ID - timestamp (str): When the error occurred - url (str): URL that caused the error - error (str): Error message * robotsBlocked (List[str]): List of URLs blocked by robots.txt Raises: Exception: If error check fails rrNrDrEs r#rz&AsyncV1FirecrawlApp.check_crawl_errorssh&'')),,| 2 2r 2 2 2          r2cK|}tj4d{V}||jd||4d{V}|d{Vcdddd{Vcdddd{VS#1d{VswxYwY dddd{VdS#1d{VswxYwYdS)rNrrC)rrrrFrWrB)r`rr\rrs r#rz AsyncV1FirecrawlApp.cancel_crawls''))(** - - - - - - -g~~&E&E&E&Ew~WW - - - - - - -[c%]]__,,,,,, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -s5(B<B2B< B! !B<$B! %B<< C CrcK|} ||jd||d{VS#t$r!}t t |d}~wwxYw)a> Check the status of an asynchronous extraction job. Args: job_id (str): The ID of the extraction job Returns: V1ExtractResponse[Any] with: * success (bool): Whether request succeeded * data (Optional[Any]): Extracted data matching schema * error (Optional[str]): Error message if any * warning (Optional[str]): Warning message if any * sources (Optional[List[str]]): Source URLs if requested Raises: ValueError: If status check fails rNrr rWrr^r0)r`rr\rs r#r z&AsyncV1FirecrawlApp.get_extract_statuss$'')) %00<55V55  % % %SVV$$ $ %%> A)A$$A)c K|} |s|std|s|std|r||}t|pg||||dt} |r|| d<|r|| d<|r|| d< ||jd| | d {VS#t$r!} tt| d } ~ wwxYw) a Initiate an asynchronous extraction job without waiting for completion. Args: urls (Optional[List[str]]): URLs to extract from prompt (Optional[str]): Custom extraction prompt schema (Optional[Any]): JSON schema/Pydantic model system_prompt (Optional[str]): System context allow_external_links (Optional[bool]): Follow external links enable_web_search (Optional[bool]): Enable web search show_sources (Optional[bool]): Include source URLs agent (Optional[Dict[str, Any]]): Agent configuration idempotency_key (Optional[str]): Unique key to prevent duplicate requests Returns: V1ExtractResponse[Any] with: * success (bool): Whether request succeeded * data (Optional[Any]): Extracted data matching schema * error (Optional[str]): Error message if any Raises: ValueError: If job initiation fails r r rqrr*rrrN) rr^rrrrrWrr0) r`rr*rerPrQrRrSrr\rrs r#r"z!AsyncV1FirecrawlApp.async_extract,scD'')) Df DBCC C BF B@AA A  6--f55F(3-$***      ,%+L "  9+8L (  *$)L ! %11<,,,   % % %SVV$$ $ %s $B.. C8CCr$r%r&r$r%r&cKi}|||d<|||d<|||d<|||||d{V}|drd|vr|S|d} ||d{V}|d d kr|S|d d kr%td |d |d dkrnt jdd{Vt ddd dS)a Generate LLMs.txt for a given URL and monitor until completion. Args: url (str): Target URL to generate LLMs.txt from max_urls (Optional[int]): Maximum URLs to process (default: 10) show_full_text (Optional[bool]): Include full text in output (default: False) experimental_stream (Optional[bool]): Enable experimental streaming Returns: V1GenerateLLMsTextStatusResponse containing: * success (bool): Whether generation completed successfully * status (str): Status of generation (processing/completed/failed) * data (Dict[str, str], optional): Generated text with fields: - llmstxt (str): Generated LLMs.txt content - llmsfulltxt (str, optional): Full version if requested * error (str, optional): Error message if generation failed * expiresAt (str): When the generated data expires Raises: Exception: If generation fails Nr4r5r)rLrrTrr]r^z#LLMs.txt generation failed. Error: rrrFr+rr*)r,rr-rrrrM) r`rJr$r%r&r.rrrs r#r/z&AsyncV1FirecrawlApp.generate_llms_textrs:   (F9   %%3F> "  *.AF* +66 ) 3 7         ||I&& $h*>*>O$ #??GGGGGGGGFh;.. !X-- [fjjQXFYFY [ [\\\!\11-"" " " " " " " " #0Ev@HTVWWW Wr2r#r6czKi}|||d<|||d<|||d<t||||}|}d|i|dd}d t|d < ||jd ||d{VS#t $r!} tt| d} ~ wwxYw) a@ Initiate an asynchronous LLMs.txt generation job without waiting for completion. Args: url (str): Target URL to generate LLMs.txt from max_urls (Optional[int]): Maximum URLs to process (default: 10) show_full_text (Optional[bool]): Include full text in output (default: False) cache (Optional[bool]): Whether to use cached content if available (default: True) experimental_stream (Optional[bool]): Enable experimental streaming Returns: V1GenerateLLMsTextResponse containing: * success (bool): Whether job started successfully * id (str): Unique identifier for the job * error (str, optional): Error message if start failed Raises: ValueError: If job initiation fails Nr4r5r)r(rJTrsrqrr1) r2rrzrrrWrr^r0) r`rJr$r%r6r&r.r\r2rs r#r,z,AsyncV1FirecrawlApp.async_generate_llms_texts&6   (F9   %%3F> "  *.AF* +)'"5    ''))CQ6;;4;#P#PQ 5G55 ( %11<,,,   % % %SVV$$ $ %s*$B B:B55B:cK|} ||jd||d{VS#t$r!}t t |d}~wwxYw)a Check the status of an asynchronous LLMs.txt generation job. Args: id (str): The ID of the generation job Returns: V1GenerateLLMsTextStatusResponse containing: * success (bool): Whether generation completed successfully * status (str): Status of generation (processing/completed/failed) * data (Dict[str, str], optional): Generated text with fields: - llmstxt (str): Generated LLMs.txt content - llmsfulltxt (str, optional): Full version if requested * error (str, optional): Error message if generation failed * expiresAt (str): When the generated data expires Raises: ValueError: If status check fails r6NrIr`rr\rs r#r-z3AsyncV1FirecrawlApp.check_generate_llms_text_statuss('')) %00<11R11  % % %SVV$$ $ %rJ)rr\r$r]rP/_AsyncV1FirecrawlApp__experimental_stream_stepsr_r`r r\r]rQr_r`cKi} ||| d<||| d<||| d<||| d<||| d<||| d<tdi| } |||||||d{V} | d rd | vr| S| d } d } d } || d{V}|r9d |vr5|d | d}|D] }||t |d } | r9d|vr5|d|d}|D] }| |t |d}|ddkr|S|ddkr%t d|d|ddkrnt jdd{VtddSrb) r9rdrrerrrrrB)r`r rr\r$r]rPrQr_r`rfrrrgrhrrirjrkrls r#rmz!AsyncV1FirecrawlApp.deep_researchsP  *3OJ '  !+5OK (  )1OI &  &0?O, -  $.;ON + & 2*>O$ #::6BBBBBBBBF @|v55!' !56I6J6J!K .**HK))))&)&*>&?&?# ;Y&00$Y/0A0B0BC )&&FIf%%%%$'y(9$:$:!h;.. !X-- U 7@S@S U UVVV!\11-"" " " " " " " "- #0,EAlmmmmr2)rr\r$r]rPrQcKi}|||d<|||d<|||d<|||d<|||d<|||d<tdi|}|} d|i|d d } d t| d < ||jd | | d{VS#t $r!} tt| d} ~ wwxYw)roNrr;r4r=rrcr Trsrqrrpr1) r9rrzrrrWrr^r0) r`r rr\r$r]rPrQrfr\r2rs r#rdz'AsyncV1FirecrawlApp.async_deep_researchcsL>  *3OJ '  !+5OK (  )1OI &  &0?O, -  $.;ON + & 2Ki} | rUt| tr| | n*| | dd||| d<||| d<||| d<||| d<||| d<||| d <||| d <| | dd| d <| | tdd |i| } | dd}d t|d<||jd|dd|jid{VS)a Asynchronously search for content using Firecrawl. Args: query (str): Search query string limit (Optional[int]): Max results (default: 5) tbs (Optional[str]): Time filter (e.g. "qdr:d") filter (Optional[str]): Custom result filter lang (Optional[str]): Language code (default: "en") country (Optional[str]): Country code (default: "us") location (Optional[str]): Geo-targeting timeout (Optional[int]): Request timeout in milliseconds scrape_options (Optional[ScrapeOptions]): Result scraping configuration params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters **kwargs: Additional keyword arguments for future compatibility Returns: SearchResponse: Response containing: * success (bool): Whether request succeeded * data (List[FirecrawlDocument]): Search results * warning (Optional[str]): Warning message if any * error (Optional[str]): Error message if any Raises: Exception: If search fails or response cannot be parsed TrsNrr rrrXrurtrr rqrrrrr1)ryrzrr rrrWrV)r`r rr rrrXrurtrr.rrrrs r#rzAsyncV1FirecrawlApp.searchsR  T&$'' T$$V,,,,$$V[[$T[%R%RSSS  %*M' " ?#&M%  &,M( #  $(M& !  '.M) $  (0M* %  '.M) $  %-;-@-@$]a-@-b-bM/ * V$$$&CCEC]CC "''D'II 7g 7 7 H--| ' ' '  6 66 7         r2)r\)Nr:r;rr)rr)Rr+r,r-r.r0rarrrrrrrr rClientResponserr#rrr"rrrrrrr rrWrrrrrrrrrrrrrpr'rrrrrir[rrrrrr8rrrrrOrrrrrr r"rMr/rGr,r-rrBrmrdrer rr __classcell__rs@r#r r  s} ;;;c;;;;;;.2$'+4+4+4+4#s(^ +4 4S>* +4  +4"+4-1cN+4+4+4+4^7:^^^"&sCx.^;?S>^^.3^>B38n^^^^27:]]]%)#s(^]].3]>B38n]]]],+G,B+C+TX++++6 Z# Zs Z[^ Zor Zwz Z Z Z Z-(=----,';,,,,/3-1 *9*9*9]+*9&c] *97L *9*9*9*9^04-1 (@(@s)(@^,(@&c] (@7L (@(@(@(@\mq04040404&*%*37%)4837(,OS(,.237EI)DMDMDMDMd7,g$hij DM d38n- DM #49- DM#49-DM (~DMsmDMc]DM/0DMTNDM$,D>DM#+4.DM ~DM G$JKL!DM" ~#DM$l+%DM&#<0'DM(d57I=ZgivyGIWYrt*$@AB)DM**#.+DMDMDMDMTW[,0,0,0,0"&!&/3!%04/3$(KO*./3AE*.'()--HCHCHC3iHC$w(Q RST HC $sCx.) HC tCy) HCtCy)HC$D>HC3-HC#HC+,HCHC (~HC'tnHCD>HC  FGH!HC",'#HC$|,%HC&$u\3E}VceruCESUnp{&{ |}~'HC(')HC* }+HC,"#-HC0 %1HCHCHCHC^W[,0,0,0,0"&!&/3!%04/3$(KO*./3AE*..2)--JIJIJI3iJI$w(Q RST JI $sCx.) JI tCy) JItCy)JI$D>JI3-JI#JI+,JIJI (~JI'tnJID>JI  FGH!JI",'#JI$|,%JI&$u\3E}VceruCESUnp{&{ |}~'JI(')JI*&d^+JI,"#-JI0 1JIJIJIJI`.2-1#'-1#/3.2/3)-489=3726,0#+/'()-+uBuBuB uB S * uB  S * uB C= uB&c]uB}uB'tnuB&d^uB'tnuB!uB!1uB%_ 456uB#+4.uB "*$!uB"$D>#uB$}%uB&#4.'uB( })uB*"#+uB. /uBuBuBuBx.2-1#'-1#/3.2/3)-489=3726,0#+/'()-+sBsBsB sB S * sB  S * sB C= sB&c]sB}sB'tnsB&d^sB'tnsB!sB!1sB%_ 456sB#+4.sB "*$!sB"$D>#sB$}%sB&#4.'sB( })sB*"#+sB. /sBsBsBsBj@3@3H@@@@D)_)_#)_S#X)__b)_lA)_)_)_)_^!%)--1'+#!&/3(,KEKEKE KE KE ! KE %TN KEtnKE}KE#KE+,KE%KE2?KEKEKEKE^)-dR%)$(+/3805+0.2dRdRdR49%dRSM dR SM dR $C= dR#+4.dR (~dR#4.dRDcN+dR8I7MdRdRdRdRLB #B :UB B B B H # :O    2 3 3H    2-S-T#s(^----(%s%7H7M%%%%:)-D%%)$(+/3805+0.2D%D%D%49%D%SM D% SM D% $C= D%#+4.D% (~D%#4.D%DcN+D%8I7MD%D%D%D%T'+-126 ;W;W;W;Wsm ;W %TN ;W "*$ ;W <\ ;W;W;W;WB'+-1$(265%5%5%5%sm 5% %TN 5% D> 5%"*$5%FJDH^n^n^n^n } ^n ! ^n sm ^n&c]^n$C=^n*2$^n"(DcN+;T+A"BC^n $sCx.)94)? @A^nNj^n^n^n^nH(,(,&*-1+/:>:%:%:%:% } :% ! :% sm :%&c]:%$C=:%*2$:%DHS>:%:%:%:%x %3 %;W % % % %L$(!%$("&%)&*%*8>#ABCN *N N N N N N N N r2r c eZdZdZdedeffd ZddZddZd e ee fddfd Z d e j d eddfd Zded edededef dZxZS)r/zQ Async version of V1CrawlWatcher that properly handles async operations. rrcLt||dSrr)r`rrrs r#razAsyncV1CrawlWatcher.__init__s# S!!!!!r2rXNcKtj|jdd|jjfg4d{V }||d{Vdddd{VdS#1d{VswxYwYdS)z[ Establishes async WebSocket connection and starts listening for messages. rr)rNrrs r#rzAsyncV1CrawlWatcher.connects%% K!02NDHres<     XXXXXXXXXXXXXXXXXXXXXX  +'+K88888    +-- GCLLJ!!!!!X'!!! (((((H.(((h( CCCCC8-CCC:::::(,gaj::: *****x)*** UUUUUh(UUUh0$$$$$h($$$(#####8%### """""+""" H& H&      H&   #####X'### X' 2 """""($"""(((((X'(((+++++8%+++....._...     *1-wqz   ,,,,,H.,,, $ $ $ $ $("4 $ $ $+++++H&+++(     h(    $ $ $ $ $H. $ $ $8%H. 0 0 0 0 0($ 0 0 0     H&   4 4 4 4 4h( 4 4 4 & & & & &*GAJ & & & 4 4 4 4 4X' 4 4 4     x)   -----*--- H.-----x)--- 8-H$6 11111h&81118#5 00000X%700011111x1111 6 6 6 6 68- 6 6 6     X/        8#5        !3   &&&&&8+=&&&x'9     x)    + + + + +h( + + +b* b* b* b* b* b* b* b* JUYRYRYRYRYRYRYRYRvh h h h h .h h h T9OZOZOZOZOZ.OZOZOZOZOZr2