[{"data":1,"prerenderedAt":290},["ShallowReactive",2],{"navigation_docs":3,"-guide-crawl":88,"-guide-crawl-surround":285},[4,48],{"title":5,"path":6,"stem":7,"children":8,"icon":47},"Guide","\u002Fguide","1.guide\u002F01.index",[9,11,15,19,23,27,31,35,39,43],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Scrape","\u002Fguide\u002Fscrape","1.guide\u002F02.scrape",{"title":16,"path":17,"stem":18},"Screenshots and PDFs","\u002Fguide\u002Fcaptures","1.guide\u002F03.captures",{"title":20,"path":21,"stem":22},"Links and accessibility and extract","\u002Fguide\u002Freading","1.guide\u002F04.reading",{"title":24,"path":25,"stem":26},"Crawl and search","\u002Fguide\u002Fcrawl","1.guide\u002F05.crawl",{"title":28,"path":29,"stem":30},"Sessions","\u002Fguide\u002Fsessions","1.guide\u002F06.sessions",{"title":32,"path":33,"stem":34},"CLI","\u002Fguide\u002Fcli","1.guide\u002F07.cli",{"title":36,"path":37,"stem":38},"MCP and Pi and OMP","\u002Fguide\u002Fagents","1.guide\u002F08.agents",{"title":40,"path":41,"stem":42},"Errors","\u002Fguide\u002Ferrors","1.guide\u002F09.errors",{"title":44,"path":45,"stem":46},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F10.custom","i-lucide-book-open",{"title":49,"path":50,"stem":51,"children":52,"icon":87},"Providers","\u002Fproviders","2.providers\u002F00.index",[53,55,59,63,67,71,75,79,83],{"title":54,"path":50,"stem":51},"Every provider",{"title":56,"path":57,"stem":58},"Steel","\u002Fproviders\u002Fsteel","2.providers\u002F01.steel",{"title":60,"path":61,"stem":62},"Browserbase","\u002Fproviders\u002Fbrowserbase","2.providers\u002F02.browserbase",{"title":64,"path":65,"stem":66},"Kernel","\u002Fproviders\u002Fkernel","2.providers\u002F03.kernel",{"title":68,"path":69,"stem":70},"Browserless","\u002Fproviders\u002Fbrowserless","2.providers\u002F04.browserless",{"title":72,"path":73,"stem":74},"Hyperbrowser","\u002Fproviders\u002Fhyperbrowser","2.providers\u002F05.hyperbrowser",{"title":76,"path":77,"stem":78},"Anchor","\u002Fproviders\u002Fanchor","2.providers\u002F06.anchor",{"title":80,"path":81,"stem":82},"Cloudflare","\u002Fproviders\u002Fcloudflare","2.providers\u002F07.cloudflare",{"title":84,"path":85,"stem":86},"Playwright","\u002Fproviders\u002Fplaywright","2.providers\u002F08.playwright","i-lucide-server",{"id":89,"title":24,"body":90,"description":277,"extension":278,"links":279,"meta":280,"navigation":282,"path":25,"seo":283,"stem":26,"__hash__":284},"docs\u002F1.guide\u002F05.crawl.md",{"type":91,"value":92,"toc":270},"minimark",[93,98,136,144,156,160,163,169,188,221,224,228,235,239,254,260,266],[94,95,97],"h2",{"id":96},"crawl","Crawl",[99,100,105],"pre",{"className":101,"code":102,"language":103,"meta":104,"style":104},"language-bash shiki shiki-themes browsers browsers browsers","browsers crawl https:\u002F\u002Fexample.com --maxPages 5 -p playwright\n","bash","",[106,107,108],"code",{"__ignoreMap":104},[109,110,113,117,121,124,127,130,133],"span",{"class":111,"line":112},"line",1,[109,114,116],{"class":115},"sK71F","browsers",[109,118,120],{"class":119},"shU9J"," crawl",[109,122,123],{"class":119}," https:\u002F\u002Fexample.com",[109,125,126],{"class":119}," --maxPages",[109,128,129],{"class":119}," 5",[109,131,132],{"class":119}," -p",[109,134,135],{"class":119}," playwright\n",[99,137,142],{"className":138,"code":140,"language":141,"meta":104},[139],"language-text","ℹ Crawling via playwright...\n\n--- https:\u002F\u002Fexample.com ---\nThis domain is for use in documentation examples without needing permission. …\n","text",[106,143,140],{"__ignoreMap":104},[145,146,147,148,151,152,155],"p",{},"A crawl starts at one page and follows links, ten pages unless you say ",[106,149,150],{},"--maxPages",", one link deep unless you say ",[106,153,154],{},"--maxDepth",". Cloudflare, Hyperbrowser and Playwright crawl. Playwright does it right there in your Chromium. The other two start a job on their side and poll it.",[94,157,159],{"id":158},"a-slow-crawl-keeps-its-job","A slow crawl keeps its job",[145,161,162],{},"A big site takes a while, and a tool call can't wait forever. Cloudflare and Hyperbrowser get two minutes. A job still running after that doesn't turn into a timeout, it comes back with its ID:",[99,164,167],{"className":165,"code":166,"language":141,"meta":104},[139],"[provider=cloudflare] Crawled 0 pages.\nJob ID: 5b1f… (status: running)\nThe job is still running. Call browsers_crawl with this jobId and provider cloudflare to wait for it again.\n",[106,168,166],{"__ignoreMap":104},[145,170,171,172,175,176,179,180,183,184,187],{},"Pass that ",[106,173,174],{},"jobId"," back, with the same provider (and the same ",[106,177,178],{},"browser"," if you used Kitesurf), and without ",[106,181,182],{},"url",". It waits for the same job. No second crawl, no second bill. In the CLI that's ",[106,185,186],{},"--job",":",[99,189,191],{"className":101,"code":190,"language":103,"meta":104,"style":104},"browsers crawl --job \u003Cjob-id> -p cloudflare\n",[106,192,193],{"__ignoreMap":104},[109,194,195,197,199,202,206,209,213,216,218],{"class":111,"line":112},[109,196,116],{"class":115},[109,198,120],{"class":119},[109,200,201],{"class":119}," --job",[109,203,205],{"class":204},"skH_V"," \u003C",[109,207,208],{"class":119},"job-i",[109,210,212],{"class":211},"s38Sx","d",[109,214,215],{"class":204},">",[109,217,132],{"class":119},[109,219,220],{"class":119}," cloudflare\n",[145,222,223],{},"A job ID belongs to the provider that made it, so naming the provider isn't optional there. Picking one on its own, the package could land on a provider that has never heard of your job.",[94,225,227],{"id":226},"shared-budget","Shared budget",[145,229,230,231,234],{},"For a model, every page's text shares one ",[106,232,233],{},"maxChars",", 20 000 by default. Ten pages don't get 20 000 each. That's deliberate. A crawl is the easiest way to drown a context window, and this one can't.",[94,236,238],{"id":237},"search","Search",[99,240,242],{"className":101,"code":241,"language":103,"meta":104,"style":104},"browsers search \"browser automation\"\n",[106,243,244],{"__ignoreMap":104},[109,245,246,248,251],{"class":111,"line":112},[109,247,116],{"class":115},[109,249,250],{"class":119}," search",[109,252,253],{"class":119}," \"browser automation\"\n",[99,255,258],{"className":256,"code":257,"language":141,"meta":104},[139],"Awesome Browser Automation - GitHub\n  https:\u002F\u002Fgithub.com\u002Fangrykoala\u002Fawesome-browser-automation\n  A curated list of awesome browser automation tools and resources. …\n",[106,259,257],{"__ignoreMap":104},[145,261,262,263,265],{},"Hyperbrowser is the one with a search route, so ",[106,264,237],{}," goes there. Title, URL and a snippet per result. It's a web search that happens to live in a browser package, and it's handy exactly when a model needs a URL before it can scrape one.",[267,268,269],"style",{},"html pre.shiki code .sK71F, html code.shiki .sK71F{--shiki-light:var(--shiki-token-function);--shiki-default:var(--shiki-token-function);--shiki-dark:var(--shiki-token-function)}html pre.shiki code .shU9J, html code.shiki .shU9J{--shiki-light:var(--shiki-token-string);--shiki-default:var(--shiki-token-string);--shiki-dark:var(--shiki-token-string)}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .skH_V, html code.shiki .skH_V{--shiki-light:var(--shiki-token-keyword);--shiki-default:var(--shiki-token-keyword);--shiki-dark:var(--shiki-token-keyword)}html pre.shiki code .s38Sx, html code.shiki .s38Sx{--shiki-light:var(--ui-text-highlighted);--shiki-default:var(--ui-text-highlighted);--shiki-dark:var(--ui-text-highlighted)}",{"title":104,"searchDepth":271,"depth":271,"links":272},2,[273,274,275,276],{"id":96,"depth":271,"text":97},{"id":158,"depth":271,"text":159},{"id":226,"depth":271,"text":227},{"id":237,"depth":271,"text":238},"Follow links from one page and keep a slow crawl job instead of losing it. Web search through Hyperbrowser","md",null,{"icon":281},"i-lucide-network",true,{"title":24,"description":277},"vd9PMLl-Dr48xK2DJgDt-euBg_FoUYG50MSKkxTmo3A",[286,288],{"title":20,"path":21,"stem":22,"description":287,"children":-1},"Three other ways to read one page. The hrefs the scrape dropped and the fields of a form and structured data from a prompt",{"title":28,"path":29,"stem":30,"description":289,"children":-1},"Open a live browser and release it. The session ID an agent gets and the CDP URL only your code gets",1790685118173]