[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"glossary-crawl-budget::en":3,"gloss-cluster-crawl-budget::en":23,"gloss-next-crawl-budget::en":9},{"slug":4,"category":5,"name":6,"definition":7,"meta_desc":8,"faq":9,"schema_markup":9,"related":10},"crawl-budget","growth","Crawl Budget","Crawl budget is the amount of crawling a search engine is willing to spend on a site — informally the combination of how fast it can fetch pages without straining the server and how much it currently wants that site's content. It is not a figure anyone publishes and not a setting you configure; it is an emergent constraint you only notice when it binds.\n\nFor most sites it never binds, and a good deal of advice about it is misapplied. A few hundred pages that change occasionally will be crawled thoroughly without anyone thinking about it. The constraint becomes real on large sites, on sites that generate addresses combinatorially, and on slow origins where each fetch is expensive enough that the engine backs off on its own.\n\nWhen it does bind, the cause is almost always waste rather than volume. Faceted navigation multiplying filters and sort orders into millions of distinct URLs. Session identifiers appended to every link. Calendars with an infinite supply of next months. Soft 404s that return a status of 200 for pages that do not exist, so every wrong address is a real page as far as the crawler can tell. Long redirect chains that spend several fetches to arrive at one document.\n\nThe fixes run in that order too. Best is not producing the URLs at all; next is disallowing the parameter space in robots.txt so the crawler never asks; next is returning honest status codes so a missing page reads as missing. Keeping the pages you care about shallow and well linked internally matters more than any of it, because a page nothing links to competes for attention from a standing start.\n\nEvidence comes from two places, neither of them a third-party tool. Your own server logs show exactly which paths were requested, by which agent, and what they received, which is the only way to see the waste directly. Each engine's own crawl statistics show the aggregate, and a rising request count against a flat number of useful pages is the signal to go looking.","Crawl budget is how much crawling an engine will spend on a site. It rarely binds — and when it does, the cause is wasted URLs rather than page count.",null,[11,14,17,20],{"slug":12,"name":13},"cdn","Content Delivery Network (CDN)",{"slug":15,"name":16},"content-optimization","Content Optimization",{"slug":18,"name":19},"programmatic-seo","Programmatic SEO",{"slug":21,"name":22},"static-site-generator","Static Site Generator (SSG)",[24,28,32,35,38,39,42,45,48,51,54,57],{"slug":25,"category":5,"name":26,"updated_at":27},"ai-crawler","AI Crawler","2026-09-08T21:32:37+00:00",{"slug":29,"category":5,"name":30,"updated_at":31},"attribution-model","Attribution Model","2026-09-08T21:32:36+00:00",{"slug":33,"category":5,"name":34,"updated_at":31},"channel-market-fit","Channel-Market Fit",{"slug":36,"category":5,"name":37,"updated_at":31},"content-brief","Content Brief",{"slug":15,"category":5,"name":16,"updated_at":31},{"slug":40,"category":5,"name":41,"updated_at":31},"conversion-rate-optimization","Conversion Rate Optimization (CRO)",{"slug":43,"category":5,"name":44,"updated_at":27},"entity-seo","Entity SEO",{"slug":46,"category":5,"name":47,"updated_at":27},"featured-snippet","Featured Snippet",{"slug":49,"category":5,"name":50,"updated_at":27},"generative-engine-optimization","Generative Engine Optimization (GEO)",{"slug":52,"category":5,"name":53,"updated_at":31},"growth-loop","Growth Loop",{"slug":55,"category":5,"name":56,"updated_at":27},"internal-linking","Internal Linking",{"slug":58,"category":5,"name":59,"updated_at":31},"jobs-to-be-done","Jobs to Be Done (JTBD)"]