[{"data":1,"prerenderedAt":3164},["ShallowReactive",2],{"navLinks":3,"sidebar_docs_navigation_\u002Fblog\u002Fthe-evolution-of-bot-management":124,"navigation":140,"navLinks_footer":181,"\u002Fblog\u002Fthe-evolution-of-bot-management_page":211,"\u002Fblog\u002Fthe-evolution-of-bot-management":1914},{"id":4,"extension":5,"links":6,"meta":121,"stem":122,"__hash__":123},"navigationMenu\u002Fnavigation.json","json",[7,57,62,103],{"label":8,"icon":9,"nested":10,"children":11},"API Products","i-lucide-box",true,[12,17,22,27,32,37,42,47,52],{"label":13,"to":14,"description":15,"icon":16},"Full IP Lookup","\u002Ffull-ip-lookup","Enrich any IP with complete intelligence","i-lucide-globe",{"label":18,"to":19,"description":20,"icon":21},"Proxy & VPN Detection","\u002Fproxy-vpn-detection","Identify proxies and VPN networks","i-lucide-shield-alert",{"label":23,"to":24,"description":25,"icon":26},"Tor Exit Nodes","\u002Ftor-exit-nodes","Block Tor anonymity network traffic","i-lucide-eye-off",{"label":28,"to":29,"description":30,"icon":31},"Granular Geolocation","\u002Fgranular-geolocation","Highly accurate location mapping","i-lucide-map-pin",{"label":33,"to":34,"description":35,"icon":36},"Threat Scoring System","\u002Fthreat-scoring-system","Unified traffic risk score","i-lucide-activity",{"label":38,"to":39,"description":40,"icon":41},"ASN & Carrier Data","\u002Fasn-carrier-data","Identify autonomous systems","i-lucide-server",{"label":43,"to":44,"description":45,"icon":46},"Disposable Email Check","\u002Fdisposable-email-check","Detect temporary email domains","i-lucide-mail",{"label":48,"to":49,"description":50,"icon":51},"User Agent Parser","\u002Fuser-agent-bot-parser","Parse messy HTTP headers","i-lucide-bot",{"label":53,"to":54,"description":55,"icon":56},"Verified Bots","\u002Fverified-bots-crawlers","Identify legitimate search engine crawlers","i-lucide-check-circle",{"label":58,"to":59,"icon":60,"nested":61},"Pricing","\u002Fpricing","i-lucide-credit-card",false,{"label":63,"to":64,"icon":65,"nested":10,"children":66},"Docs","\u002Fdocs","i-lucide-book-open",[67,71,76,81,86,91,95,98],{"label":68,"to":64,"description":69,"icon":70},"Getting Started","Learn how to setup and use IP Shield","i-lucide-rocket",{"label":72,"to":73,"description":74,"icon":75},"All Endpoints","\u002Fdocs\u002Fendpoints\u002Femail-domain-check","Reference for all IP Shield REST API endpoints","i-lucide-plug",{"label":77,"to":78,"description":79,"icon":80},"Network API","\u002Fdocs\u002Fendpoints\u002Fnetwork","Network intelligence and routing data","i-lucide-network",{"label":82,"to":83,"description":84,"icon":85},"Summary API","\u002Fdocs\u002Fendpoints\u002Fsummary","Full IP intelligence summary","i-lucide-bar-chart-2",{"label":87,"to":88,"description":89,"icon":90},"Lookup IP API","\u002Fdocs\u002Fendpoints\u002Flookup-ip","Single IP data lookup","i-lucide-search",{"label":92,"to":93,"description":94,"icon":31},"Geo Check API","\u002Fdocs\u002Fendpoints\u002Fgeo-check","Check geographic location",{"label":96,"to":73,"description":97,"icon":46},"Email Domain Check API","Verify disposable email domains",{"label":99,"to":100,"description":101,"icon":102},"User Agent Check API","\u002Fdocs\u002Fendpoints\u002Fuser-agent-check","Detect malicious user agents","i-lucide-monitor",{"label":104,"icon":105,"nested":10,"children":106},"Resources","i-lucide-library",[107,112,117],{"label":108,"to":109,"icon":110,"description":111},"About Us","\u002Fabout","i-lucide-info","Learn more about IP Shield and our mission",{"label":113,"to":114,"icon":115,"description":116},"Blog","\u002Fblog","i-lucide-newspaper","Latest news, updates, and security articles",{"label":118,"to":119,"description":120,"icon":46},"Contact Us","\u002Fcontact","Get in touch with our support team",{},"navigation","OH-Zj4UmYqVQJC8Ts6f4KeKRzXJYi5trdK22JpX9Y0Q",[125],{"title":113,"path":114,"stem":126,"children":127,"page":61},"blog",[128,132,136],{"title":129,"path":130,"stem":131},"Combating Residential Proxies and VPNs in Modern Applications","\u002Fblog\u002Fcombating-residential-proxies","blog\u002Fcombating-residential-proxies",{"title":133,"path":134,"stem":135},"Ultimate Guide to Trading Automation & Volume Strategies","\u002Fblog\u002Fintroducing-ip-shield","blog\u002Fintroducing-ip-shield",{"title":137,"path":138,"stem":139},"The Evolution of Bot Management and User Agent Parsing","\u002Fblog\u002Fthe-evolution-of-bot-management","blog\u002Fthe-evolution-of-bot-management",[141],{"title":63,"path":64,"stem":142,"children":143},"docs",[144,147,164],{"title":145,"path":64,"stem":146},"Introduction","docs\u002Findex",{"title":148,"path":149,"stem":150,"children":151,"page":61},"Credit & Authorization","\u002Fdocs\u002Fcredits-and-auth","docs\u002Fcredits-and-auth",[152,156,160],{"title":153,"path":154,"stem":155},"Authentication","\u002Fdocs\u002Fcredits-and-auth\u002Fauth","docs\u002Fcredits-and-auth\u002Fauth",{"title":157,"path":158,"stem":159},"Credits & Usage","\u002Fdocs\u002Fcredits-and-auth\u002Fcredits-usage","docs\u002Fcredits-and-auth\u002Fcredits-usage",{"title":161,"path":162,"stem":163},"Rate Limits","\u002Fdocs\u002Fcredits-and-auth\u002Frate-limits","docs\u002Fcredits-and-auth\u002Frate-limits",{"title":165,"path":166,"stem":167,"children":168,"page":61},"API Endpoints","\u002Fdocs\u002Fendpoints","docs\u002Fendpoints",[169,171,173,175,177,179],{"title":96,"path":73,"stem":170},"docs\u002Fendpoints\u002Femail-domain-check",{"title":92,"path":93,"stem":172},"docs\u002Fendpoints\u002Fgeo-check",{"title":87,"path":88,"stem":174},"docs\u002Fendpoints\u002Flookup-ip",{"title":77,"path":78,"stem":176},"docs\u002Fendpoints\u002Fnetwork",{"title":82,"path":83,"stem":178},"docs\u002Fendpoints\u002Fsummary",{"title":99,"path":100,"stem":180},"docs\u002Fendpoints\u002Fuser-agent-check",{"id":4,"extension":5,"links":182,"meta":210,"stem":122,"__hash__":123},[183,194,195,205],{"label":8,"icon":9,"nested":10,"children":184},[185,186,187,188,189,190,191,192,193],{"label":13,"to":14,"description":15,"icon":16},{"label":18,"to":19,"description":20,"icon":21},{"label":23,"to":24,"description":25,"icon":26},{"label":28,"to":29,"description":30,"icon":31},{"label":33,"to":34,"description":35,"icon":36},{"label":38,"to":39,"description":40,"icon":41},{"label":43,"to":44,"description":45,"icon":46},{"label":48,"to":49,"description":50,"icon":51},{"label":53,"to":54,"description":55,"icon":56},{"label":58,"to":59,"icon":60,"nested":61},{"label":63,"to":64,"icon":65,"nested":10,"children":196},[197,198,199,200,201,202,203,204],{"label":68,"to":64,"description":69,"icon":70},{"label":72,"to":73,"description":74,"icon":75},{"label":77,"to":78,"description":79,"icon":80},{"label":82,"to":83,"description":84,"icon":85},{"label":87,"to":88,"description":89,"icon":90},{"label":92,"to":93,"description":94,"icon":31},{"label":96,"to":73,"description":97,"icon":46},{"label":99,"to":100,"description":101,"icon":102},{"label":104,"icon":105,"nested":10,"children":206},[207,208,209],{"label":108,"to":109,"icon":110,"description":111},{"label":113,"to":114,"icon":115,"description":116},{"label":118,"to":119,"description":120,"icon":46},{},{"id":212,"title":137,"author":213,"authorGithub":214,"authorGithubUserName":215,"authorImg":216,"body":217,"date":1898,"description":1899,"extension":1900,"featured":61,"icon":1901,"image":1902,"meta":1903,"navigation":10,"path":138,"rawbody":1904,"readingTime":1905,"seo":1906,"stem":139,"tags":1907,"__hash__":1913},"blog\u002Fblog\u002Fthe-evolution-of-bot-management.md","Sergio","https:\u002F\u002Fgithub.com\u002FSergo706","Sergo706","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F187537278?v=4",{"type":218,"value":219,"toc":1880},"minimark",[220,224,228,248,251,257,262,265,270,287,291,298,302,309,313,316,423,429,433,436,447,450,454,457,505,508,512,515,518,522,525,1521,1527,1531,1534,1773,1776,1820,1824,1827,1833,1836,1840,1863,1867,1870,1873,1876],[221,222,137],"h1",{"id":223},"the-evolution-of-bot-management-and-user-agent-parsing",[225,226,227],"p",{},"Bot traffic comprises nearly half of all internet traffic today. While some bots index your content for search engines, others scrape your proprietary data, scalp your inventory, probe your applications for vulnerabilities, or consume massive amounts of your bandwidth. Managing this traffic effectively determines the performance, cost-efficiency, and security of your digital infrastructure.",[225,229,230,231,235,236,239,240,243,244,247],{},"In the early days of the web, bot management was incredibly straightforward. It meant reading the ",[232,233,234],"code",{},"User-Agent"," HTTP header. Developers simply wrote regular expressions to block strings containing ",[232,237,238],{},"python-requests",", ",[232,241,242],{},"curl",", or ",[232,245,246],{},"Java\u002F1.8.0",". Today, malicious actors spoof their headers to perfectly mimic Chrome running on a macOS device, rendering naive string matching obsolete.",[225,249,250],{},"This comprehensive guide explores the multi-generational evolution of bot architecture, why legacy detection mechanisms fail, and how to implement modern, context-aware bot mitigation strategies using IP Shield and advanced server-side architectures.",[252,253,254],"note",{},[225,255,256],{},"A User-Agent string identifies the client software originating the request. Because the client controls this header entirely, it provides no inherent cryptographic proof of identity. Relying on it for security is equivalent to trusting a visitor's handwritten name tag.",[258,259,261],"h2",{"id":260},"the-generations-of-web-scraping","The Generations of Web Scraping",[225,263,264],{},"To understand how to defeat modern bots, we must trace their evolutionary history. As defensive mechanisms improved, scraper technology evolved in direct response, creating a fascinating arms race.",[266,267,269],"h3",{"id":268},"generation-1-the-scripted-clients","Generation 1: The Scripted Clients",[225,271,272,273,239,276,279,280,283,284,286],{},"The earliest bots were simple HTTP clients. Tools like ",[232,274,275],{},"cURL",[232,277,278],{},"wget",", and libraries like ",[232,281,282],{},"urllib"," or ",[232,285,238],{}," made bare-bones HTTP GET and POST requests. They were extremely fast and efficient but completely incapable of executing JavaScript. If a website rendered content client-side via React or Angular, Generation 1 bots failed entirely. They were also easily detected by their default User-Agent strings.",[266,288,290],{"id":289},"generation-2-the-early-headless-browsers","Generation 2: The Early Headless Browsers",[225,292,293,294,297],{},"As the web moved towards Single Page Applications (SPAs), bots needed to execute JavaScript. Projects like PhantomJS emerged, providing a scriptable, headless WebKit engine. While they could render JS, they leaked massive amounts of identifiable information. Security tools easily detected them by checking for variables like ",[232,295,296],{},"window._phantom"," or analyzing their unique TLS fingerprint.",[266,299,301],{"id":300},"generation-3-the-modern-headless-era","Generation 3: The Modern Headless Era",[225,303,304,305,308],{},"Today, attackers utilize headless versions of modern browsers controlled via frameworks like Puppeteer, Playwright, and Selenium. These tools execute JavaScript, solve simple CAPTCHAs, and mimic human interaction patterns perfectly. Furthermore, developers have created \"stealth\" plugins (e.g., ",[232,306,307],{},"puppeteer-extra-plugin-stealth",") designed specifically to patch the JavaScript variables and inconsistencies that anti-bot software looks for. They render pages just like a real user, making detection incredibly difficult at the application layer.",[266,310,312],{"id":311},"generation-4-ai-agents-and-distributed-scrapers","Generation 4: AI Agents and Distributed Scrapers",[225,314,315],{},"We are currently entering the fourth generation. Bots are no longer simple scripts; they are AI-driven agents powered by Large Language Models (LLMs) that can navigate complex dynamic UIs, understand semantic page layouts, and dynamically adjust their scraping strategies when website structures change. When an attacker pairs these AI agents with a rotating residential proxy network, the traffic looks indistinguishable from organic human user growth.",[317,318,323],"pre",{"className":319,"code":320,"language":321,"meta":322,"style":322},"language-mermaid shiki shiki-themes light-plus light-plus dracula","sequenceDiagram\n    participant AI as AI Scraper Agent\n    participant Proxy as Residential Proxy Pool\n    participant WAF as Web Application Firewall\n    participant App as Your Application\n\n    AI->>Proxy: Request Page (Spoofed Chrome UA)\n    Proxy->>WAF: Forward Request via Home IP\n    WAF->>WAF: Check IP Reputation (Passes)\n    WAF->>WAF: Check User-Agent (Passes)\n    WAF->>App: Forward Request\n    App-->>WAF: Return HTML payload\n    WAF-->>Proxy: Return HTML payload\n    Proxy-->>AI: Return HTML payload\n    AI->>AI: LLM parses unstructured DOM\n    AI->>Proxy: Proceed to next logical step\n","mermaid","",[232,324,325,333,339,345,351,357,363,369,375,381,387,393,399,405,411,417],{"__ignoreMap":322},[326,327,330],"span",{"class":328,"line":329},"line",1,[326,331,332],{},"sequenceDiagram\n",[326,334,336],{"class":328,"line":335},2,[326,337,338],{},"    participant AI as AI Scraper Agent\n",[326,340,342],{"class":328,"line":341},3,[326,343,344],{},"    participant Proxy as Residential Proxy Pool\n",[326,346,348],{"class":328,"line":347},4,[326,349,350],{},"    participant WAF as Web Application Firewall\n",[326,352,354],{"class":328,"line":353},5,[326,355,356],{},"    participant App as Your Application\n",[326,358,360],{"class":328,"line":359},6,[326,361,362],{"emptyLinePlaceholder":10},"\n",[326,364,366],{"class":328,"line":365},7,[326,367,368],{},"    AI->>Proxy: Request Page (Spoofed Chrome UA)\n",[326,370,372],{"class":328,"line":371},8,[326,373,374],{},"    Proxy->>WAF: Forward Request via Home IP\n",[326,376,378],{"class":328,"line":377},9,[326,379,380],{},"    WAF->>WAF: Check IP Reputation (Passes)\n",[326,382,384],{"class":328,"line":383},10,[326,385,386],{},"    WAF->>WAF: Check User-Agent (Passes)\n",[326,388,390],{"class":328,"line":389},11,[326,391,392],{},"    WAF->>App: Forward Request\n",[326,394,396],{"class":328,"line":395},12,[326,397,398],{},"    App-->>WAF: Return HTML payload\n",[326,400,402],{"class":328,"line":401},13,[326,403,404],{},"    WAF-->>Proxy: Return HTML payload\n",[326,406,408],{"class":328,"line":407},14,[326,409,410],{},"    Proxy-->>AI: Return HTML payload\n",[326,412,414],{"class":328,"line":413},15,[326,415,416],{},"    AI->>AI: LLM parses unstructured DOM\n",[326,418,420],{"class":328,"line":419},16,[326,421,422],{},"    AI->>Proxy: Proceed to next logical step\n",[424,425,426],"warning",{},[225,427,428],{},"Relying solely on User-Agent parsing to block bots leaves your application completely exposed to modern headless scraping frameworks and AI agents. You must look beyond the header.",[258,430,432],{"id":431},"verifying-legitimate-bots-the-seo-dilemma","Verifying Legitimate Bots: The SEO Dilemma",[225,434,435],{},"Not all automated traffic is harmful. Search engine crawlers like Googlebot, Bingbot, Yandex, and specialized SEO tools (Ahrefs, Semrush) must access your site to ensure your business remains visible online. You cannot afford to block them accidentally with overly aggressive anti-bot rules; doing so destroys your organic search rankings.",[225,437,438,439,442,443,446],{},"However, malicious scrapers frequently spoof the ",[232,440,441],{},"Googlebot"," User-Agent string to bypass security filters. They know that naive WAF configurations include rules like: ",[232,444,445],{},"IF User-Agent CONTAINS \"Googlebot\" THEN ALLOW",".",[225,448,449],{},"You must verify that the bot claiming to be Google actually originates from an IP address owned by Google. This requires cross-referencing the claimed identity with the underlying network infrastructure.",[266,451,453],{"id":452},"the-verification-process-reverse-dns","The Verification Process: Reverse DNS",[225,455,456],{},"The traditional way to verify a search engine crawler is performing a Reverse DNS (rDNS) lookup followed by a Forward DNS lookup.",[458,459,460,475,487,496],"ol",{},[461,462,463,467,468,471,472,446],"li",{},[464,465,466],"strong",{},"Reverse DNS Lookup",": You query the DNS pointer (PTR) record of the incoming IP address. For example, doing an rDNS lookup on ",[232,469,470],{},"66.249.66.1"," returns ",[232,473,474],{},"crawl-66-249-66-1.googlebot.com",[461,476,477,480,481,283,484,446],{},[464,478,479],{},"Domain Verification",": You verify that the domain ends in ",[232,482,483],{},"googlebot.com",[232,485,486],{},"google.com",[461,488,489,492,493,495],{},[464,490,491],{},"Forward DNS Lookup",": To prevent an attacker from simply setting up a fake PTR record on their own server, you must do a forward DNS lookup on the domain returned in step 1 (",[232,494,474],{},").",[461,497,498,501,502,504],{},[464,499,500],{},"Final Match",": If the IP returned in step 3 matches the original IP ",[232,503,470],{},", the crawler is verified.",[225,506,507],{},"Performing this three-step DNS dance for every single incoming request introduces massive latency to your application. It is computationally expensive and slow.",[266,509,511],{"id":510},"the-modern-solution-ip-shield-bot-api","The Modern Solution: IP Shield Bot API",[225,513,514],{},"IP Shield simplifies this verification process dramatically. The User Agent & Bot Parser API analyzes the header string and automatically performs the reverse DNS lookups, ASN verification, and signature matching in real-time. It explicitly flags whether a known crawler is verified or spoofed.",[225,516,517],{},"You implement this check in your edge routing layer or middleware. This ensures fake crawlers get dropped before they consume your application resources, while legitimate SEO bots pass through smoothly.",[258,519,521],{"id":520},"implementing-bot-protection-middleware","Implementing Bot Protection Middleware",[225,523,524],{},"Here is a comprehensive example of verifying search engine crawlers and blocking malicious bots using a Nuxt 3 server middleware. This implementation handles rate limiting, safe-listing, spoof detection, and adaptive responses.",[317,526,531],{"className":527,"code":528,"filename":529,"language":530,"meta":322,"style":322},"language-typescript shiki shiki-themes light-plus light-plus dracula","import { IPShield } from '@ip-shield\u002Fsdk';\nimport { sendError, setResponseStatus, createError } from 'h3';\n\n\u002F\u002F Initialize the SDK\nconst shield = new IPShield(process.env.IP_SHIELD_API_KEY);\n\n\u002F\u002F Cache verified IPs in memory to reduce API calls and latency\n\u002F\u002F In production, use Redis or unstorage for distributed caching\nconst verifiedBotCache = new Set\u003Cstring>();\n\nexport default defineEventHandler(async (event) => {\n  \u002F\u002F Extract essential request metadata\n  const ip = getRequestIP(event, { xForwardedFor: true }) || '';\n  const userAgent = getRequestHeader(event, 'user-agent') || '';\n  const path = event.path;\n  \n  \u002F\u002F Skip static assets to save compute resources\n  if (path.startsWith('\u002F_nuxt\u002F') || path.match(\u002F\\.(js|css|png|jpg|svg|ico)$\u002F)) {\n    return;\n  }\n\n  \u002F\u002F Fast path: if we've already verified this IP recently, let it through\n  if (verifiedBotCache.has(ip)) {\n    return;\n  }\n  \n  try {\n    \u002F\u002F Cross-reference the IP and User-Agent with IP Shield\n    const botData = await shield.bots.analyze({ ip, userAgent });\n    \n    \u002F\u002F Scenario 1: The request claims to be a good bot but the IP doesn't match\n    if (botData.isBot && botData.isSpoofed) {\n      console.warn(`Spoofed bot detected from IP: ${ip} claiming to be: ${userAgent}`);\n      \n      \u002F\u002F We drop spoofed bots immediately with a 403 Forbidden.\n      setResponseStatus(event, 403);\n      return { \n        error: 'spoofed_identity_detected',\n        message: 'Your network origin does not match your claimed identity.'\n      };\n    }\n    \n    \u002F\u002F Scenario 2: It is a verified, legitimate search engine crawler\n    if (botData.isVerifiedCrawler) {\n      \u002F\u002F Cache the IP to speed up subsequent requests from this crawler\n      verifiedBotCache.add(ip);\n      \n      \u002F\u002F Optionally limit the cache size to prevent memory leaks\n      if (verifiedBotCache.size > 10000) verifiedBotCache.clear();\n      \n      return; \u002F\u002F Allow the request to proceed\n    }\n    \n    \u002F\u002F Scenario 3: It is an unverified, generic scraper script (e.g., python-requests)\n    \u002F\u002F We don't want these consuming our server rendering resources.\n    if (botData.isBot && !botData.isVerifiedCrawler) {\n      setResponseStatus(event, 429);\n      return { \n        error: 'automated_traffic_blocked',\n        message: 'Automated access is restricted. Please use our official API.'\n      };\n    }\n\n    \u002F\u002F Scenario 4: It appears to be a legitimate human user.\n    \u002F\u002F Proceed to the next middleware or route handler.\n    return;\n    \n  } catch (err) {\n    \u002F\u002F Fail open: if the IP Shield API is down, we don't want to block organic traffic\n    console.error('Bot analysis failed:', err);\n    return;\n  }\n});\n","server\u002Fmiddleware\u002Fbot-protection.ts","typescript",[232,532,533,567,598,602,608,650,654,659,664,688,692,724,729,771,804,823,828,834,924,932,938,943,949,971,978,983,988,996,1002,1041,1047,1053,1082,1120,1126,1132,1150,1159,1177,1193,1199,1205,1210,1216,1232,1238,1255,1260,1266,1299,1304,1315,1320,1325,1331,1337,1363,1379,1386,1402,1416,1421,1426,1431,1437,1443,1450,1455,1471,1477,1503,1510,1515],{"__ignoreMap":322},[326,534,535,539,543,547,550,553,557,561,564],{"class":328,"line":329},[326,536,538],{"class":537},"sZ328","import",[326,540,542],{"class":541},"sDd4n"," { ",[326,544,546],{"class":545},"sjsA6","IPShield",[326,548,549],{"class":541}," } ",[326,551,552],{"class":537},"from",[326,554,556],{"class":555},"sFkSl"," '",[326,558,560],{"class":559},"sFB1V","@ip-shield\u002Fsdk",[326,562,563],{"class":555},"'",[326,565,566],{"class":541},";\n",[326,568,569,571,573,576,578,581,583,586,588,590,592,594,596],{"class":328,"line":335},[326,570,538],{"class":537},[326,572,542],{"class":541},[326,574,575],{"class":545},"sendError",[326,577,239],{"class":541},[326,579,580],{"class":545},"setResponseStatus",[326,582,239],{"class":541},[326,584,585],{"class":545},"createError",[326,587,549],{"class":541},[326,589,552],{"class":537},[326,591,556],{"class":555},[326,593,266],{"class":559},[326,595,563],{"class":555},[326,597,566],{"class":541},[326,599,600],{"class":328,"line":341},[326,601,362],{"emptyLinePlaceholder":10},[326,603,604],{"class":328,"line":347},[326,605,607],{"class":606},"sghk6","\u002F\u002F Initialize the SDK\n",[326,609,610,614,618,622,626,630,633,636,638,641,643,647],{"class":328,"line":353},[326,611,613],{"class":612},"sl46w","const",[326,615,617],{"class":616},"s3JHE"," shield",[326,619,621],{"class":620},"saOXh"," =",[326,623,625],{"class":624},"sakC6"," new",[326,627,629],{"class":628},"sHOzp"," IPShield",[326,631,632],{"class":541},"(",[326,634,635],{"class":545},"process",[326,637,446],{"class":541},[326,639,640],{"class":545},"env",[326,642,446],{"class":541},[326,644,646],{"class":645},"sPzPf","IP_SHIELD_API_KEY",[326,648,649],{"class":541},");\n",[326,651,652],{"class":328,"line":359},[326,653,362],{"emptyLinePlaceholder":10},[326,655,656],{"class":328,"line":365},[326,657,658],{"class":606},"\u002F\u002F Cache verified IPs in memory to reduce API calls and latency\n",[326,660,661],{"class":328,"line":371},[326,662,663],{"class":606},"\u002F\u002F In production, use Redis or unstorage for distributed caching\n",[326,665,666,668,671,673,675,678,681,685],{"class":328,"line":377},[326,667,613],{"class":612},[326,669,670],{"class":616}," verifiedBotCache",[326,672,621],{"class":620},[326,674,625],{"class":624},[326,676,677],{"class":628}," Set",[326,679,680],{"class":541},"\u003C",[326,682,684],{"class":683},"sFs1U","string",[326,686,687],{"class":541},">();\n",[326,689,690],{"class":328,"line":383},[326,691,362],{"emptyLinePlaceholder":10},[326,693,694,697,700,703,705,708,711,715,718,721],{"class":328,"line":389},[326,695,696],{"class":537},"export",[326,698,699],{"class":537}," default",[326,701,702],{"class":628}," defineEventHandler",[326,704,632],{"class":541},[326,706,707],{"class":612},"async",[326,709,710],{"class":541}," (",[326,712,714],{"class":713},"sygFZ","event",[326,716,717],{"class":541},") ",[326,719,720],{"class":612},"=>",[326,722,723],{"class":541}," {\n",[326,725,726],{"class":328,"line":395},[326,727,728],{"class":606},"  \u002F\u002F Extract essential request metadata\n",[326,730,731,734,737,739,742,744,746,749,752,756,760,763,766,769],{"class":328,"line":401},[326,732,733],{"class":612},"  const",[326,735,736],{"class":616}," ip",[326,738,621],{"class":620},[326,740,741],{"class":628}," getRequestIP",[326,743,632],{"class":541},[326,745,714],{"class":545},[326,747,748],{"class":541},", { ",[326,750,751],{"class":545},"xForwardedFor",[326,753,755],{"class":754},"s34zl",":",[326,757,759],{"class":758},"sjR7W"," true",[326,761,762],{"class":541}," }) ",[326,764,765],{"class":620},"||",[326,767,768],{"class":555}," ''",[326,770,566],{"class":541},[326,772,773,775,778,780,783,785,787,789,791,794,796,798,800,802],{"class":328,"line":407},[326,774,733],{"class":612},[326,776,777],{"class":616}," userAgent",[326,779,621],{"class":620},[326,781,782],{"class":628}," getRequestHeader",[326,784,632],{"class":541},[326,786,714],{"class":545},[326,788,239],{"class":541},[326,790,563],{"class":555},[326,792,793],{"class":559},"user-agent",[326,795,563],{"class":555},[326,797,717],{"class":541},[326,799,765],{"class":620},[326,801,768],{"class":555},[326,803,566],{"class":541},[326,805,806,808,811,813,816,818,821],{"class":328,"line":413},[326,807,733],{"class":612},[326,809,810],{"class":616}," path",[326,812,621],{"class":620},[326,814,815],{"class":545}," event",[326,817,446],{"class":541},[326,819,820],{"class":545},"path",[326,822,566],{"class":541},[326,824,825],{"class":328,"line":419},[326,826,827],{"class":541},"  \n",[326,829,831],{"class":328,"line":830},17,[326,832,833],{"class":606},"  \u002F\u002F Skip static assets to save compute resources\n",[326,835,837,840,842,844,846,849,851,853,856,858,860,862,864,866,869,871,875,879,882,886,890,893,895,898,900,903,905,908,910,913,916,919,921],{"class":328,"line":836},18,[326,838,839],{"class":537},"  if",[326,841,710],{"class":541},[326,843,820],{"class":545},[326,845,446],{"class":541},[326,847,848],{"class":628},"startsWith",[326,850,632],{"class":541},[326,852,563],{"class":555},[326,854,855],{"class":559},"\u002F_nuxt\u002F",[326,857,563],{"class":555},[326,859,717],{"class":541},[326,861,765],{"class":620},[326,863,810],{"class":545},[326,865,446],{"class":541},[326,867,868],{"class":628},"match",[326,870,632],{"class":541},[326,872,874],{"class":873},"sRg35","\u002F",[326,876,878],{"class":877},"skAEd","\\.",[326,880,632],{"class":881},"s63D3",[326,883,885],{"class":884},"sIcdS","js",[326,887,889],{"class":888},"st6lo","|",[326,891,892],{"class":884},"css",[326,894,889],{"class":888},[326,896,897],{"class":884},"png",[326,899,889],{"class":888},[326,901,902],{"class":884},"jpg",[326,904,889],{"class":888},[326,906,907],{"class":884},"svg",[326,909,889],{"class":888},[326,911,912],{"class":884},"ico",[326,914,915],{"class":881},")",[326,917,918],{"class":888},"$",[326,920,874],{"class":873},[326,922,923],{"class":541},")) {\n",[326,925,927,930],{"class":328,"line":926},19,[326,928,929],{"class":537},"    return",[326,931,566],{"class":541},[326,933,935],{"class":328,"line":934},20,[326,936,937],{"class":541},"  }\n",[326,939,941],{"class":328,"line":940},21,[326,942,362],{"emptyLinePlaceholder":10},[326,944,946],{"class":328,"line":945},22,[326,947,948],{"class":606},"  \u002F\u002F Fast path: if we've already verified this IP recently, let it through\n",[326,950,952,954,956,959,961,964,966,969],{"class":328,"line":951},23,[326,953,839],{"class":537},[326,955,710],{"class":541},[326,957,958],{"class":545},"verifiedBotCache",[326,960,446],{"class":541},[326,962,963],{"class":628},"has",[326,965,632],{"class":541},[326,967,968],{"class":545},"ip",[326,970,923],{"class":541},[326,972,974,976],{"class":328,"line":973},24,[326,975,929],{"class":537},[326,977,566],{"class":541},[326,979,981],{"class":328,"line":980},25,[326,982,937],{"class":541},[326,984,986],{"class":328,"line":985},26,[326,987,827],{"class":541},[326,989,991,994],{"class":328,"line":990},27,[326,992,993],{"class":537},"  try",[326,995,723],{"class":541},[326,997,999],{"class":328,"line":998},28,[326,1000,1001],{"class":606},"    \u002F\u002F Cross-reference the IP and User-Agent with IP Shield\n",[326,1003,1005,1008,1011,1013,1016,1018,1020,1023,1025,1028,1031,1033,1035,1038],{"class":328,"line":1004},29,[326,1006,1007],{"class":612},"    const",[326,1009,1010],{"class":616}," botData",[326,1012,621],{"class":620},[326,1014,1015],{"class":537}," await",[326,1017,617],{"class":545},[326,1019,446],{"class":541},[326,1021,1022],{"class":545},"bots",[326,1024,446],{"class":541},[326,1026,1027],{"class":628},"analyze",[326,1029,1030],{"class":541},"({ ",[326,1032,968],{"class":545},[326,1034,239],{"class":541},[326,1036,1037],{"class":545},"userAgent",[326,1039,1040],{"class":541}," });\n",[326,1042,1044],{"class":328,"line":1043},30,[326,1045,1046],{"class":541},"    \n",[326,1048,1050],{"class":328,"line":1049},31,[326,1051,1052],{"class":606},"    \u002F\u002F Scenario 1: The request claims to be a good bot but the IP doesn't match\n",[326,1054,1056,1059,1061,1064,1066,1069,1072,1074,1076,1079],{"class":328,"line":1055},32,[326,1057,1058],{"class":537},"    if",[326,1060,710],{"class":541},[326,1062,1063],{"class":545},"botData",[326,1065,446],{"class":541},[326,1067,1068],{"class":545},"isBot",[326,1070,1071],{"class":620}," &&",[326,1073,1010],{"class":545},[326,1075,446],{"class":541},[326,1077,1078],{"class":545},"isSpoofed",[326,1080,1081],{"class":541},") {\n",[326,1083,1085,1088,1090,1093,1095,1098,1101,1103,1106,1109,1111,1113,1115,1118],{"class":328,"line":1084},33,[326,1086,1087],{"class":545},"      console",[326,1089,446],{"class":541},[326,1091,1092],{"class":628},"warn",[326,1094,632],{"class":541},[326,1096,1097],{"class":559},"`Spoofed bot detected from IP: ",[326,1099,1100],{"class":612},"${",[326,1102,968],{"class":545},[326,1104,1105],{"class":612},"}",[326,1107,1108],{"class":559}," claiming to be: ",[326,1110,1100],{"class":612},[326,1112,1037],{"class":545},[326,1114,1105],{"class":612},[326,1116,1117],{"class":559},"`",[326,1119,649],{"class":541},[326,1121,1123],{"class":328,"line":1122},34,[326,1124,1125],{"class":541},"      \n",[326,1127,1129],{"class":328,"line":1128},35,[326,1130,1131],{"class":606},"      \u002F\u002F We drop spoofed bots immediately with a 403 Forbidden.\n",[326,1133,1135,1138,1140,1142,1144,1148],{"class":328,"line":1134},36,[326,1136,1137],{"class":628},"      setResponseStatus",[326,1139,632],{"class":541},[326,1141,714],{"class":545},[326,1143,239],{"class":541},[326,1145,1147],{"class":1146},"spgvN","403",[326,1149,649],{"class":541},[326,1151,1153,1156],{"class":328,"line":1152},37,[326,1154,1155],{"class":537},"      return",[326,1157,1158],{"class":541}," { \n",[326,1160,1162,1165,1167,1169,1172,1174],{"class":328,"line":1161},38,[326,1163,1164],{"class":545},"        error",[326,1166,755],{"class":754},[326,1168,556],{"class":555},[326,1170,1171],{"class":559},"spoofed_identity_detected",[326,1173,563],{"class":555},[326,1175,1176],{"class":541},",\n",[326,1178,1180,1183,1185,1187,1190],{"class":328,"line":1179},39,[326,1181,1182],{"class":545},"        message",[326,1184,755],{"class":754},[326,1186,556],{"class":555},[326,1188,1189],{"class":559},"Your network origin does not match your claimed identity.",[326,1191,1192],{"class":555},"'\n",[326,1194,1196],{"class":328,"line":1195},40,[326,1197,1198],{"class":541},"      };\n",[326,1200,1202],{"class":328,"line":1201},41,[326,1203,1204],{"class":541},"    }\n",[326,1206,1208],{"class":328,"line":1207},42,[326,1209,1046],{"class":541},[326,1211,1213],{"class":328,"line":1212},43,[326,1214,1215],{"class":606},"    \u002F\u002F Scenario 2: It is a verified, legitimate search engine crawler\n",[326,1217,1219,1221,1223,1225,1227,1230],{"class":328,"line":1218},44,[326,1220,1058],{"class":537},[326,1222,710],{"class":541},[326,1224,1063],{"class":545},[326,1226,446],{"class":541},[326,1228,1229],{"class":545},"isVerifiedCrawler",[326,1231,1081],{"class":541},[326,1233,1235],{"class":328,"line":1234},45,[326,1236,1237],{"class":606},"      \u002F\u002F Cache the IP to speed up subsequent requests from this crawler\n",[326,1239,1241,1244,1246,1249,1251,1253],{"class":328,"line":1240},46,[326,1242,1243],{"class":545},"      verifiedBotCache",[326,1245,446],{"class":541},[326,1247,1248],{"class":628},"add",[326,1250,632],{"class":541},[326,1252,968],{"class":545},[326,1254,649],{"class":541},[326,1256,1258],{"class":328,"line":1257},47,[326,1259,1125],{"class":541},[326,1261,1263],{"class":328,"line":1262},48,[326,1264,1265],{"class":606},"      \u002F\u002F Optionally limit the cache size to prevent memory leaks\n",[326,1267,1269,1272,1274,1276,1278,1281,1284,1287,1289,1291,1293,1296],{"class":328,"line":1268},49,[326,1270,1271],{"class":537},"      if",[326,1273,710],{"class":541},[326,1275,958],{"class":545},[326,1277,446],{"class":541},[326,1279,1280],{"class":545},"size",[326,1282,1283],{"class":620}," >",[326,1285,1286],{"class":1146}," 10000",[326,1288,717],{"class":541},[326,1290,958],{"class":545},[326,1292,446],{"class":541},[326,1294,1295],{"class":628},"clear",[326,1297,1298],{"class":541},"();\n",[326,1300,1302],{"class":328,"line":1301},50,[326,1303,1125],{"class":541},[326,1305,1307,1309,1312],{"class":328,"line":1306},51,[326,1308,1155],{"class":537},[326,1310,1311],{"class":541},"; ",[326,1313,1314],{"class":606},"\u002F\u002F Allow the request to proceed\n",[326,1316,1318],{"class":328,"line":1317},52,[326,1319,1204],{"class":541},[326,1321,1323],{"class":328,"line":1322},53,[326,1324,1046],{"class":541},[326,1326,1328],{"class":328,"line":1327},54,[326,1329,1330],{"class":606},"    \u002F\u002F Scenario 3: It is an unverified, generic scraper script (e.g., python-requests)\n",[326,1332,1334],{"class":328,"line":1333},55,[326,1335,1336],{"class":606},"    \u002F\u002F We don't want these consuming our server rendering resources.\n",[326,1338,1340,1342,1344,1346,1348,1350,1352,1355,1357,1359,1361],{"class":328,"line":1339},56,[326,1341,1058],{"class":537},[326,1343,710],{"class":541},[326,1345,1063],{"class":545},[326,1347,446],{"class":541},[326,1349,1068],{"class":545},[326,1351,1071],{"class":620},[326,1353,1354],{"class":620}," !",[326,1356,1063],{"class":545},[326,1358,446],{"class":541},[326,1360,1229],{"class":545},[326,1362,1081],{"class":541},[326,1364,1366,1368,1370,1372,1374,1377],{"class":328,"line":1365},57,[326,1367,1137],{"class":628},[326,1369,632],{"class":541},[326,1371,714],{"class":545},[326,1373,239],{"class":541},[326,1375,1376],{"class":1146},"429",[326,1378,649],{"class":541},[326,1380,1382,1384],{"class":328,"line":1381},58,[326,1383,1155],{"class":537},[326,1385,1158],{"class":541},[326,1387,1389,1391,1393,1395,1398,1400],{"class":328,"line":1388},59,[326,1390,1164],{"class":545},[326,1392,755],{"class":754},[326,1394,556],{"class":555},[326,1396,1397],{"class":559},"automated_traffic_blocked",[326,1399,563],{"class":555},[326,1401,1176],{"class":541},[326,1403,1405,1407,1409,1411,1414],{"class":328,"line":1404},60,[326,1406,1182],{"class":545},[326,1408,755],{"class":754},[326,1410,556],{"class":555},[326,1412,1413],{"class":559},"Automated access is restricted. Please use our official API.",[326,1415,1192],{"class":555},[326,1417,1419],{"class":328,"line":1418},61,[326,1420,1198],{"class":541},[326,1422,1424],{"class":328,"line":1423},62,[326,1425,1204],{"class":541},[326,1427,1429],{"class":328,"line":1428},63,[326,1430,362],{"emptyLinePlaceholder":10},[326,1432,1434],{"class":328,"line":1433},64,[326,1435,1436],{"class":606},"    \u002F\u002F Scenario 4: It appears to be a legitimate human user.\n",[326,1438,1440],{"class":328,"line":1439},65,[326,1441,1442],{"class":606},"    \u002F\u002F Proceed to the next middleware or route handler.\n",[326,1444,1446,1448],{"class":328,"line":1445},66,[326,1447,929],{"class":537},[326,1449,566],{"class":541},[326,1451,1453],{"class":328,"line":1452},67,[326,1454,1046],{"class":541},[326,1456,1458,1461,1464,1466,1469],{"class":328,"line":1457},68,[326,1459,1460],{"class":541},"  } ",[326,1462,1463],{"class":537},"catch",[326,1465,710],{"class":541},[326,1467,1468],{"class":545},"err",[326,1470,1081],{"class":541},[326,1472,1474],{"class":328,"line":1473},69,[326,1475,1476],{"class":606},"    \u002F\u002F Fail open: if the IP Shield API is down, we don't want to block organic traffic\n",[326,1478,1480,1483,1485,1488,1490,1492,1495,1497,1499,1501],{"class":328,"line":1479},70,[326,1481,1482],{"class":545},"    console",[326,1484,446],{"class":541},[326,1486,1487],{"class":628},"error",[326,1489,632],{"class":541},[326,1491,563],{"class":555},[326,1493,1494],{"class":559},"Bot analysis failed:",[326,1496,563],{"class":555},[326,1498,239],{"class":541},[326,1500,1468],{"class":545},[326,1502,649],{"class":541},[326,1504,1506,1508],{"class":328,"line":1505},71,[326,1507,929],{"class":537},[326,1509,566],{"class":541},[326,1511,1513],{"class":328,"line":1512},72,[326,1514,937],{"class":541},[326,1516,1518],{"class":328,"line":1517},73,[326,1519,1520],{"class":541},"});\n",[1522,1523,1524],"important",{},[225,1525,1526],{},"Always deploy bot protection logic as early in the request lifecycle as possible. Blocking requests at the edge or middleware layer prevents expensive database queries, business logic execution, and Server-Side Rendering (SSR) cycles from running unnecessarily.",[258,1528,1530],{"id":1529},"advanced-implementations-python-fastapi","Advanced Implementations: Python FastAPI",[225,1532,1533],{},"For data science teams or Python-heavy backends, integrating IP Shield into FastAPI is highly effective for protecting API endpoints from scrapers.",[317,1535,1540],{"className":1536,"code":1537,"filename":1538,"language":1539,"meta":322,"style":322},"language-python shiki shiki-themes light-plus light-plus dracula","from fastapi import Request, HTTPException, status\nimport httpx\nimport os\n\nIP_SHIELD_API_KEY = os.getenv(\"IP_SHIELD_API_KEY\")\n\nasync def verify_bot_traffic(request: Request):\n    \"\"\"\n    FastAPI dependency to intercept and analyze incoming traffic.\n    \"\"\"\n    client_ip = request.client.host\n    user_agent = request.headers.get('user-agent', '')\n    \n    # Handle proxy headers if behind an ingress controller\n    forwarded_for = request.headers.get('x-forwarded-for')\n    if forwarded_for:\n        client_ip = forwarded_for.split(',')[0].strip()\n\n    async with httpx.AsyncClient() as client:\n        try:\n            response = await client.post(\n                \"https:\u002F\u002Fip-shield.riavzon.com\u002Fv1\u002Fcheck\u002Fbot\",\n                json={\"ip\": client_ip, \"userAgent\": user_agent},\n                headers={\"Authorization\": f\"Bearer {IP_SHIELD_API_KEY}\"},\n                timeout=2.0\n            )\n            \n            if response.status_code == 200:\n                data = response.json()\n                \n                if data.get(\"isSpoofed\"):\n                    raise HTTPException(\n                        status_code=status.HTTP_403_FORBIDDEN,\n                        detail=\"Spoofed User-Agent identity detected.\"\n                    )\n                    \n                if data.get(\"isBot\") and not data.get(\"isVerifiedCrawler\"):\n                    raise HTTPException(\n                        status_code=status.HTTP_429_TOO_MANY_REQUESTS,\n                        detail=\"Automated access restricted.\"\n                    )\n                    \n        except httpx.RequestError as exc:\n            # Fail open on timeout or network error\n            print(f\"IP Shield validation error: {exc}\")\n            pass\n            \n    return True\n","app\u002Fdependencies\u002Fbot_protection.py","python",[232,1541,1542,1547,1552,1557,1561,1566,1570,1575,1580,1585,1589,1594,1599,1603,1608,1613,1618,1623,1627,1632,1637,1642,1647,1652,1657,1662,1667,1672,1677,1682,1687,1692,1697,1702,1707,1712,1717,1722,1726,1731,1736,1740,1744,1749,1754,1759,1764,1768],{"__ignoreMap":322},[326,1543,1544],{"class":328,"line":329},[326,1545,1546],{},"from fastapi import Request, HTTPException, status\n",[326,1548,1549],{"class":328,"line":335},[326,1550,1551],{},"import httpx\n",[326,1553,1554],{"class":328,"line":341},[326,1555,1556],{},"import os\n",[326,1558,1559],{"class":328,"line":347},[326,1560,362],{"emptyLinePlaceholder":10},[326,1562,1563],{"class":328,"line":353},[326,1564,1565],{},"IP_SHIELD_API_KEY = os.getenv(\"IP_SHIELD_API_KEY\")\n",[326,1567,1568],{"class":328,"line":359},[326,1569,362],{"emptyLinePlaceholder":10},[326,1571,1572],{"class":328,"line":365},[326,1573,1574],{},"async def verify_bot_traffic(request: Request):\n",[326,1576,1577],{"class":328,"line":371},[326,1578,1579],{},"    \"\"\"\n",[326,1581,1582],{"class":328,"line":377},[326,1583,1584],{},"    FastAPI dependency to intercept and analyze incoming traffic.\n",[326,1586,1587],{"class":328,"line":383},[326,1588,1579],{},[326,1590,1591],{"class":328,"line":389},[326,1592,1593],{},"    client_ip = request.client.host\n",[326,1595,1596],{"class":328,"line":395},[326,1597,1598],{},"    user_agent = request.headers.get('user-agent', '')\n",[326,1600,1601],{"class":328,"line":401},[326,1602,1046],{},[326,1604,1605],{"class":328,"line":407},[326,1606,1607],{},"    # Handle proxy headers if behind an ingress controller\n",[326,1609,1610],{"class":328,"line":413},[326,1611,1612],{},"    forwarded_for = request.headers.get('x-forwarded-for')\n",[326,1614,1615],{"class":328,"line":419},[326,1616,1617],{},"    if forwarded_for:\n",[326,1619,1620],{"class":328,"line":830},[326,1621,1622],{},"        client_ip = forwarded_for.split(',')[0].strip()\n",[326,1624,1625],{"class":328,"line":836},[326,1626,362],{"emptyLinePlaceholder":10},[326,1628,1629],{"class":328,"line":926},[326,1630,1631],{},"    async with httpx.AsyncClient() as client:\n",[326,1633,1634],{"class":328,"line":934},[326,1635,1636],{},"        try:\n",[326,1638,1639],{"class":328,"line":940},[326,1640,1641],{},"            response = await client.post(\n",[326,1643,1644],{"class":328,"line":945},[326,1645,1646],{},"                \"https:\u002F\u002Fip-shield.riavzon.com\u002Fv1\u002Fcheck\u002Fbot\",\n",[326,1648,1649],{"class":328,"line":951},[326,1650,1651],{},"                json={\"ip\": client_ip, \"userAgent\": user_agent},\n",[326,1653,1654],{"class":328,"line":973},[326,1655,1656],{},"                headers={\"Authorization\": f\"Bearer {IP_SHIELD_API_KEY}\"},\n",[326,1658,1659],{"class":328,"line":980},[326,1660,1661],{},"                timeout=2.0\n",[326,1663,1664],{"class":328,"line":985},[326,1665,1666],{},"            )\n",[326,1668,1669],{"class":328,"line":990},[326,1670,1671],{},"            \n",[326,1673,1674],{"class":328,"line":998},[326,1675,1676],{},"            if response.status_code == 200:\n",[326,1678,1679],{"class":328,"line":1004},[326,1680,1681],{},"                data = response.json()\n",[326,1683,1684],{"class":328,"line":1043},[326,1685,1686],{},"                \n",[326,1688,1689],{"class":328,"line":1049},[326,1690,1691],{},"                if data.get(\"isSpoofed\"):\n",[326,1693,1694],{"class":328,"line":1055},[326,1695,1696],{},"                    raise HTTPException(\n",[326,1698,1699],{"class":328,"line":1084},[326,1700,1701],{},"                        status_code=status.HTTP_403_FORBIDDEN,\n",[326,1703,1704],{"class":328,"line":1122},[326,1705,1706],{},"                        detail=\"Spoofed User-Agent identity detected.\"\n",[326,1708,1709],{"class":328,"line":1128},[326,1710,1711],{},"                    )\n",[326,1713,1714],{"class":328,"line":1134},[326,1715,1716],{},"                    \n",[326,1718,1719],{"class":328,"line":1152},[326,1720,1721],{},"                if data.get(\"isBot\") and not data.get(\"isVerifiedCrawler\"):\n",[326,1723,1724],{"class":328,"line":1161},[326,1725,1696],{},[326,1727,1728],{"class":328,"line":1179},[326,1729,1730],{},"                        status_code=status.HTTP_429_TOO_MANY_REQUESTS,\n",[326,1732,1733],{"class":328,"line":1195},[326,1734,1735],{},"                        detail=\"Automated access restricted.\"\n",[326,1737,1738],{"class":328,"line":1201},[326,1739,1711],{},[326,1741,1742],{"class":328,"line":1207},[326,1743,1716],{},[326,1745,1746],{"class":328,"line":1212},[326,1747,1748],{},"        except httpx.RequestError as exc:\n",[326,1750,1751],{"class":328,"line":1218},[326,1752,1753],{},"            # Fail open on timeout or network error\n",[326,1755,1756],{"class":328,"line":1234},[326,1757,1758],{},"            print(f\"IP Shield validation error: {exc}\")\n",[326,1760,1761],{"class":328,"line":1240},[326,1762,1763],{},"            pass\n",[326,1765,1766],{"class":328,"line":1257},[326,1767,1671],{},[326,1769,1770],{"class":328,"line":1262},[326,1771,1772],{},"    return True\n",[225,1774,1775],{},"You then inject this dependency into your highly targeted routes:",[317,1777,1780],{"className":1536,"code":1778,"filename":1779,"language":1539,"meta":322,"style":322},"from fastapi import FastAPI, Depends\nfrom app.dependencies.bot_protection import verify_bot_traffic\n\napp = FastAPI()\n\n@app.get(\"\u002Fapi\u002Fv1\u002Fsensitive-data\", dependencies=[Depends(verify_bot_traffic)])\nasync def get_sensitive_data():\n    return {\"data\": \"This data is protected from unauthorized scrapers.\"}\n","app\u002Fmain.py",[232,1781,1782,1787,1792,1796,1801,1805,1810,1815],{"__ignoreMap":322},[326,1783,1784],{"class":328,"line":329},[326,1785,1786],{},"from fastapi import FastAPI, Depends\n",[326,1788,1789],{"class":328,"line":335},[326,1790,1791],{},"from app.dependencies.bot_protection import verify_bot_traffic\n",[326,1793,1794],{"class":328,"line":341},[326,1795,362],{"emptyLinePlaceholder":10},[326,1797,1798],{"class":328,"line":347},[326,1799,1800],{},"app = FastAPI()\n",[326,1802,1803],{"class":328,"line":353},[326,1804,362],{"emptyLinePlaceholder":10},[326,1806,1807],{"class":328,"line":359},[326,1808,1809],{},"@app.get(\"\u002Fapi\u002Fv1\u002Fsensitive-data\", dependencies=[Depends(verify_bot_traffic)])\n",[326,1811,1812],{"class":328,"line":365},[326,1813,1814],{},"async def get_sensitive_data():\n",[326,1816,1817],{"class":328,"line":371},[326,1818,1819],{},"    return {\"data\": \"This data is protected from unauthorized scrapers.\"}\n",[258,1821,1823],{"id":1822},"handling-false-positives-with-adaptive-friction","Handling False Positives with Adaptive Friction",[225,1825,1826],{},"No detection system is perfect. Occasionally, a legitimate user might be using a niche browser extension, a privacy-focused corporate VPN, or an outdated operating system that triggers a high bot-probability score.",[225,1828,1829,1830,446],{},"If you institute a hard block (returning a 403 Forbidden page), you permanently lose that user. The modern approach utilizes ",[464,1831,1832],{},"Adaptive Friction",[225,1834,1835],{},"When a request's threat score falls into a \"gray area\" (e.g., highly suspicious, but not definitively proven to be a malicious script), you intercept the request and present a challenge.",[266,1837,1839],{"id":1838},"types-of-adaptive-challenges","Types of Adaptive Challenges:",[458,1841,1842,1848,1854],{},[461,1843,1844,1847],{},[464,1845,1846],{},"Visual CAPTCHAs",": The traditional approach (reCAPTCHA, hCaptcha). Effective, but heavily degrades the user experience and lowers conversion rates.",[461,1849,1850,1853],{},[464,1851,1852],{},"Proof of Work (PoW)",": Your server sends a cryptographic puzzle to the client's browser. The browser must spend a few seconds of CPU time computing the hash before it is allowed to proceed. Legitimate users barely notice the slight delay, but an attacker trying to scrape 10,000 pages per minute finds their operation crippled by CPU costs.",[461,1855,1856,1859,1860,1862],{},[464,1857,1858],{},"JavaScript Execution Challenges",": The server returns an obfuscated JavaScript payload that must execute and return a specific token. Since Generation 1 bots (like ",[232,1861,275],{},") cannot execute JavaScript, they fail immediately without bothering human users.",[258,1864,1866],{"id":1865},"conclusion","Conclusion",[225,1868,1869],{},"Effective bot management requires looking at the holistic picture. You analyze the IP reputation, the ASN, the behavioral patterns, the reverse DNS records, and the HTTP headers collectively. IP Shield aggregates these vast, complex signals into a unified threat intelligence layer.",[225,1871,1872],{},"This unified approach allows you to set dynamic, highly granular thresholds. You block requests with a definitive spoofing flag outright, challenge suspicious gray-area requests with invisible Proof of Work puzzles, and allow verified SEO traffic seamless, accelerated access.",[225,1874,1875],{},"The evolution of bot management has moved definitively away from static string-matching rules. To protect modern infrastructure, engineering teams must embrace dynamic, context-aware intelligence networks.",[1877,1878,1879],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZ328, html code.shiki .sZ328{--shiki-light:#AF00DB;--shiki-default:#AF00DB;--shiki-dark:#FF79C6}html pre.shiki code .sDd4n, html code.shiki .sDd4n{--shiki-light:#000000;--shiki-default:#000000;--shiki-dark:#F8F8F2}html pre.shiki code .sjsA6, html code.shiki .sjsA6{--shiki-light:#001080;--shiki-default:#001080;--shiki-dark:#F8F8F2}html pre.shiki code .sFkSl, html code.shiki .sFkSl{--shiki-light:#A31515;--shiki-default:#A31515;--shiki-dark:#E9F284}html pre.shiki code .sFB1V, html code.shiki .sFB1V{--shiki-light:#A31515;--shiki-default:#A31515;--shiki-dark:#F1FA8C}html pre.shiki code .sghk6, html code.shiki .sghk6{--shiki-light:#008000;--shiki-default:#008000;--shiki-dark:#6272A4}html pre.shiki code .sl46w, html code.shiki .sl46w{--shiki-light:#0000FF;--shiki-default:#0000FF;--shiki-dark:#FF79C6}html pre.shiki code .s3JHE, html code.shiki .s3JHE{--shiki-light:#0070C1;--shiki-default:#0070C1;--shiki-dark:#F8F8F2}html pre.shiki code .saOXh, html code.shiki .saOXh{--shiki-light:#000000;--shiki-default:#000000;--shiki-dark:#FF79C6}html pre.shiki code .sakC6, html code.shiki .sakC6{--shiki-light:#0000FF;--shiki-light-font-weight:inherit;--shiki-default:#0000FF;--shiki-default-font-weight:inherit;--shiki-dark:#FF79C6;--shiki-dark-font-weight:bold}html pre.shiki code .sHOzp, html code.shiki .sHOzp{--shiki-light:#795E26;--shiki-default:#795E26;--shiki-dark:#50FA7B}html pre.shiki code .sPzPf, html code.shiki .sPzPf{--shiki-light:#0070C1;--shiki-default:#0070C1;--shiki-dark:#BD93F9}html pre.shiki code .sFs1U, html code.shiki .sFs1U{--shiki-light:#267F99;--shiki-light-font-style:inherit;--shiki-default:#267F99;--shiki-default-font-style:inherit;--shiki-dark:#8BE9FD;--shiki-dark-font-style:italic}html pre.shiki code .sygFZ, html code.shiki .sygFZ{--shiki-light:#001080;--shiki-light-font-style:inherit;--shiki-default:#001080;--shiki-default-font-style:inherit;--shiki-dark:#FFB86C;--shiki-dark-font-style:italic}html pre.shiki code .s34zl, html code.shiki .s34zl{--shiki-light:#001080;--shiki-default:#001080;--shiki-dark:#FF79C6}html pre.shiki code .sjR7W, html code.shiki .sjR7W{--shiki-light:#0000FF;--shiki-default:#0000FF;--shiki-dark:#BD93F9}html pre.shiki code .sRg35, html code.shiki .sRg35{--shiki-light:#811F3F;--shiki-default:#811F3F;--shiki-dark:#FF5555}html pre.shiki code .skAEd, html code.shiki .skAEd{--shiki-light:#EE0000;--shiki-default:#EE0000;--shiki-dark:#F1FA8C}html pre.shiki code .s63D3, html code.shiki .s63D3{--shiki-light:#D16969;--shiki-default:#D16969;--shiki-dark:#FFB86C}html pre.shiki code .sIcdS, html code.shiki .sIcdS{--shiki-light:#811F3F;--shiki-default:#811F3F;--shiki-dark:#F1FA8C}html pre.shiki code .st6lo, html code.shiki .st6lo{--shiki-light:#EE0000;--shiki-default:#EE0000;--shiki-dark:#FF79C6}html pre.shiki code .spgvN, html code.shiki .spgvN{--shiki-light:#098658;--shiki-default:#098658;--shiki-dark:#BD93F9}",{"title":322,"searchDepth":335,"depth":335,"links":1881},[1882,1888,1892,1893,1894,1897],{"id":260,"depth":335,"text":261,"children":1883},[1884,1885,1886,1887],{"id":268,"depth":341,"text":269},{"id":289,"depth":341,"text":290},{"id":300,"depth":341,"text":301},{"id":311,"depth":341,"text":312},{"id":431,"depth":335,"text":432,"children":1889},[1890,1891],{"id":452,"depth":341,"text":453},{"id":510,"depth":341,"text":511},{"id":520,"depth":335,"text":521},{"id":1529,"depth":335,"text":1530},{"id":1822,"depth":335,"text":1823,"children":1895},[1896],{"id":1838,"depth":341,"text":1839},{"id":1865,"depth":335,"text":1866},"2026-06-11","An exhaustive technical deep-dive into how bot management has evolved from simple User-Agent parsing to complex behavioral analysis, reverse DNS verification, and AI scraper mitigation.","md",null,"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1485827404703-89b55fcc595e?auto=format&fit=crop&q=80&w=800",{},"---\ntitle: The Evolution of Bot Management and User Agent Parsing\ndescription: An exhaustive technical deep-dive into how bot management has evolved from simple User-Agent parsing to complex behavioral analysis, reverse DNS verification, and AI scraper mitigation.\ntags: [\"engineering\", \"bot-management\", \"seo\", \"architecture\", \"web-scraping\"]\nimage: \"https:\u002F\u002Fimages.unsplash.com\u002Fphoto-1485827404703-89b55fcc595e?auto=format&fit=crop&q=80&w=800\"\nauthor: \"Sergio\"\nauthorImg: \"https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F187537278?v=4\"\nauthorGithub: \"https:\u002F\u002Fgithub.com\u002FSergo706\"\nauthorGithubUserName: \"Sergo706\"\nfeatured: false\ndate: 2026-06-11T00:00:00.000Z\nreadingTime: \"40 min\"\n---\n\n# The Evolution of Bot Management and User Agent Parsing\n\nBot traffic comprises nearly half of all internet traffic today. While some bots index your content for search engines, others scrape your proprietary data, scalp your inventory, probe your applications for vulnerabilities, or consume massive amounts of your bandwidth. Managing this traffic effectively determines the performance, cost-efficiency, and security of your digital infrastructure.\n\nIn the early days of the web, bot management was incredibly straightforward. It meant reading the `User-Agent` HTTP header. Developers simply wrote regular expressions to block strings containing `python-requests`, `curl`, or `Java\u002F1.8.0`. Today, malicious actors spoof their headers to perfectly mimic Chrome running on a macOS device, rendering naive string matching obsolete.\n\nThis comprehensive guide explores the multi-generational evolution of bot architecture, why legacy detection mechanisms fail, and how to implement modern, context-aware bot mitigation strategies using IP Shield and advanced server-side architectures.\n\n::note\nA User-Agent string identifies the client software originating the request. Because the client controls this header entirely, it provides no inherent cryptographic proof of identity. Relying on it for security is equivalent to trusting a visitor's handwritten name tag.\n::\n\n## The Generations of Web Scraping\n\nTo understand how to defeat modern bots, we must trace their evolutionary history. As defensive mechanisms improved, scraper technology evolved in direct response, creating a fascinating arms race.\n\n### Generation 1: The Scripted Clients\nThe earliest bots were simple HTTP clients. Tools like `cURL`, `wget`, and libraries like `urllib` or `python-requests` made bare-bones HTTP GET and POST requests. They were extremely fast and efficient but completely incapable of executing JavaScript. If a website rendered content client-side via React or Angular, Generation 1 bots failed entirely. They were also easily detected by their default User-Agent strings.\n\n### Generation 2: The Early Headless Browsers\nAs the web moved towards Single Page Applications (SPAs), bots needed to execute JavaScript. Projects like PhantomJS emerged, providing a scriptable, headless WebKit engine. While they could render JS, they leaked massive amounts of identifiable information. Security tools easily detected them by checking for variables like `window._phantom` or analyzing their unique TLS fingerprint.\n\n### Generation 3: The Modern Headless Era\nToday, attackers utilize headless versions of modern browsers controlled via frameworks like Puppeteer, Playwright, and Selenium. These tools execute JavaScript, solve simple CAPTCHAs, and mimic human interaction patterns perfectly. Furthermore, developers have created \"stealth\" plugins (e.g., `puppeteer-extra-plugin-stealth`) designed specifically to patch the JavaScript variables and inconsistencies that anti-bot software looks for. They render pages just like a real user, making detection incredibly difficult at the application layer.\n\n### Generation 4: AI Agents and Distributed Scrapers\nWe are currently entering the fourth generation. Bots are no longer simple scripts; they are AI-driven agents powered by Large Language Models (LLMs) that can navigate complex dynamic UIs, understand semantic page layouts, and dynamically adjust their scraping strategies when website structures change. When an attacker pairs these AI agents with a rotating residential proxy network, the traffic looks indistinguishable from organic human user growth.\n\n```mermaid\nsequenceDiagram\n    participant AI as AI Scraper Agent\n    participant Proxy as Residential Proxy Pool\n    participant WAF as Web Application Firewall\n    participant App as Your Application\n\n    AI->>Proxy: Request Page (Spoofed Chrome UA)\n    Proxy->>WAF: Forward Request via Home IP\n    WAF->>WAF: Check IP Reputation (Passes)\n    WAF->>WAF: Check User-Agent (Passes)\n    WAF->>App: Forward Request\n    App-->>WAF: Return HTML payload\n    WAF-->>Proxy: Return HTML payload\n    Proxy-->>AI: Return HTML payload\n    AI->>AI: LLM parses unstructured DOM\n    AI->>Proxy: Proceed to next logical step\n```\n\n::warning\nRelying solely on User-Agent parsing to block bots leaves your application completely exposed to modern headless scraping frameworks and AI agents. You must look beyond the header.\n::\n\n## Verifying Legitimate Bots: The SEO Dilemma\n\nNot all automated traffic is harmful. Search engine crawlers like Googlebot, Bingbot, Yandex, and specialized SEO tools (Ahrefs, Semrush) must access your site to ensure your business remains visible online. You cannot afford to block them accidentally with overly aggressive anti-bot rules; doing so destroys your organic search rankings.\n\nHowever, malicious scrapers frequently spoof the `Googlebot` User-Agent string to bypass security filters. They know that naive WAF configurations include rules like: `IF User-Agent CONTAINS \"Googlebot\" THEN ALLOW`. \n\nYou must verify that the bot claiming to be Google actually originates from an IP address owned by Google. This requires cross-referencing the claimed identity with the underlying network infrastructure.\n\n### The Verification Process: Reverse DNS\n\nThe traditional way to verify a search engine crawler is performing a Reverse DNS (rDNS) lookup followed by a Forward DNS lookup.\n\n1. **Reverse DNS Lookup**: You query the DNS pointer (PTR) record of the incoming IP address. For example, doing an rDNS lookup on `66.249.66.1` returns `crawl-66-249-66-1.googlebot.com`.\n2. **Domain Verification**: You verify that the domain ends in `googlebot.com` or `google.com`.\n3. **Forward DNS Lookup**: To prevent an attacker from simply setting up a fake PTR record on their own server, you must do a forward DNS lookup on the domain returned in step 1 (`crawl-66-249-66-1.googlebot.com`).\n4. **Final Match**: If the IP returned in step 3 matches the original IP `66.249.66.1`, the crawler is verified.\n\nPerforming this three-step DNS dance for every single incoming request introduces massive latency to your application. It is computationally expensive and slow.\n\n### The Modern Solution: IP Shield Bot API\n\nIP Shield simplifies this verification process dramatically. The User Agent & Bot Parser API analyzes the header string and automatically performs the reverse DNS lookups, ASN verification, and signature matching in real-time. It explicitly flags whether a known crawler is verified or spoofed.\n\nYou implement this check in your edge routing layer or middleware. This ensures fake crawlers get dropped before they consume your application resources, while legitimate SEO bots pass through smoothly.\n\n## Implementing Bot Protection Middleware\n\nHere is a comprehensive example of verifying search engine crawlers and blocking malicious bots using a Nuxt 3 server middleware. This implementation handles rate limiting, safe-listing, spoof detection, and adaptive responses.\n\n~~~typescript [server\u002Fmiddleware\u002Fbot-protection.ts]\nimport { IPShield } from '@ip-shield\u002Fsdk';\nimport { sendError, setResponseStatus, createError } from 'h3';\n\n\u002F\u002F Initialize the SDK\nconst shield = new IPShield(process.env.IP_SHIELD_API_KEY);\n\n\u002F\u002F Cache verified IPs in memory to reduce API calls and latency\n\u002F\u002F In production, use Redis or unstorage for distributed caching\nconst verifiedBotCache = new Set\u003Cstring>();\n\nexport default defineEventHandler(async (event) => {\n  \u002F\u002F Extract essential request metadata\n  const ip = getRequestIP(event, { xForwardedFor: true }) || '';\n  const userAgent = getRequestHeader(event, 'user-agent') || '';\n  const path = event.path;\n  \n  \u002F\u002F Skip static assets to save compute resources\n  if (path.startsWith('\u002F_nuxt\u002F') || path.match(\u002F\\.(js|css|png|jpg|svg|ico)$\u002F)) {\n    return;\n  }\n\n  \u002F\u002F Fast path: if we've already verified this IP recently, let it through\n  if (verifiedBotCache.has(ip)) {\n    return;\n  }\n  \n  try {\n    \u002F\u002F Cross-reference the IP and User-Agent with IP Shield\n    const botData = await shield.bots.analyze({ ip, userAgent });\n    \n    \u002F\u002F Scenario 1: The request claims to be a good bot but the IP doesn't match\n    if (botData.isBot && botData.isSpoofed) {\n      console.warn(`Spoofed bot detected from IP: ${ip} claiming to be: ${userAgent}`);\n      \n      \u002F\u002F We drop spoofed bots immediately with a 403 Forbidden.\n      setResponseStatus(event, 403);\n      return { \n        error: 'spoofed_identity_detected',\n        message: 'Your network origin does not match your claimed identity.'\n      };\n    }\n    \n    \u002F\u002F Scenario 2: It is a verified, legitimate search engine crawler\n    if (botData.isVerifiedCrawler) {\n      \u002F\u002F Cache the IP to speed up subsequent requests from this crawler\n      verifiedBotCache.add(ip);\n      \n      \u002F\u002F Optionally limit the cache size to prevent memory leaks\n      if (verifiedBotCache.size > 10000) verifiedBotCache.clear();\n      \n      return; \u002F\u002F Allow the request to proceed\n    }\n    \n    \u002F\u002F Scenario 3: It is an unverified, generic scraper script (e.g., python-requests)\n    \u002F\u002F We don't want these consuming our server rendering resources.\n    if (botData.isBot && !botData.isVerifiedCrawler) {\n      setResponseStatus(event, 429);\n      return { \n        error: 'automated_traffic_blocked',\n        message: 'Automated access is restricted. Please use our official API.'\n      };\n    }\n\n    \u002F\u002F Scenario 4: It appears to be a legitimate human user.\n    \u002F\u002F Proceed to the next middleware or route handler.\n    return;\n    \n  } catch (err) {\n    \u002F\u002F Fail open: if the IP Shield API is down, we don't want to block organic traffic\n    console.error('Bot analysis failed:', err);\n    return;\n  }\n});\n~~~\n\n::important\nAlways deploy bot protection logic as early in the request lifecycle as possible. Blocking requests at the edge or middleware layer prevents expensive database queries, business logic execution, and Server-Side Rendering (SSR) cycles from running unnecessarily.\n::\n\n## Advanced Implementations: Python FastAPI\n\nFor data science teams or Python-heavy backends, integrating IP Shield into FastAPI is highly effective for protecting API endpoints from scrapers.\n\n~~~python [app\u002Fdependencies\u002Fbot_protection.py]\nfrom fastapi import Request, HTTPException, status\nimport httpx\nimport os\n\nIP_SHIELD_API_KEY = os.getenv(\"IP_SHIELD_API_KEY\")\n\nasync def verify_bot_traffic(request: Request):\n    \"\"\"\n    FastAPI dependency to intercept and analyze incoming traffic.\n    \"\"\"\n    client_ip = request.client.host\n    user_agent = request.headers.get('user-agent', '')\n    \n    # Handle proxy headers if behind an ingress controller\n    forwarded_for = request.headers.get('x-forwarded-for')\n    if forwarded_for:\n        client_ip = forwarded_for.split(',')[0].strip()\n\n    async with httpx.AsyncClient() as client:\n        try:\n            response = await client.post(\n                \"https:\u002F\u002Fip-shield.riavzon.com\u002Fv1\u002Fcheck\u002Fbot\",\n                json={\"ip\": client_ip, \"userAgent\": user_agent},\n                headers={\"Authorization\": f\"Bearer {IP_SHIELD_API_KEY}\"},\n                timeout=2.0\n            )\n            \n            if response.status_code == 200:\n                data = response.json()\n                \n                if data.get(\"isSpoofed\"):\n                    raise HTTPException(\n                        status_code=status.HTTP_403_FORBIDDEN,\n                        detail=\"Spoofed User-Agent identity detected.\"\n                    )\n                    \n                if data.get(\"isBot\") and not data.get(\"isVerifiedCrawler\"):\n                    raise HTTPException(\n                        status_code=status.HTTP_429_TOO_MANY_REQUESTS,\n                        detail=\"Automated access restricted.\"\n                    )\n                    \n        except httpx.RequestError as exc:\n            # Fail open on timeout or network error\n            print(f\"IP Shield validation error: {exc}\")\n            pass\n            \n    return True\n~~~\n\nYou then inject this dependency into your highly targeted routes:\n\n~~~python [app\u002Fmain.py]\nfrom fastapi import FastAPI, Depends\nfrom app.dependencies.bot_protection import verify_bot_traffic\n\napp = FastAPI()\n\n@app.get(\"\u002Fapi\u002Fv1\u002Fsensitive-data\", dependencies=[Depends(verify_bot_traffic)])\nasync def get_sensitive_data():\n    return {\"data\": \"This data is protected from unauthorized scrapers.\"}\n~~~\n\n## Handling False Positives with Adaptive Friction\n\nNo detection system is perfect. Occasionally, a legitimate user might be using a niche browser extension, a privacy-focused corporate VPN, or an outdated operating system that triggers a high bot-probability score. \n\nIf you institute a hard block (returning a 403 Forbidden page), you permanently lose that user. The modern approach utilizes **Adaptive Friction**.\n\nWhen a request's threat score falls into a \"gray area\" (e.g., highly suspicious, but not definitively proven to be a malicious script), you intercept the request and present a challenge.\n\n### Types of Adaptive Challenges:\n\n1. **Visual CAPTCHAs**: The traditional approach (reCAPTCHA, hCaptcha). Effective, but heavily degrades the user experience and lowers conversion rates.\n2. **Proof of Work (PoW)**: Your server sends a cryptographic puzzle to the client's browser. The browser must spend a few seconds of CPU time computing the hash before it is allowed to proceed. Legitimate users barely notice the slight delay, but an attacker trying to scrape 10,000 pages per minute finds their operation crippled by CPU costs.\n3. **JavaScript Execution Challenges**: The server returns an obfuscated JavaScript payload that must execute and return a specific token. Since Generation 1 bots (like `cURL`) cannot execute JavaScript, they fail immediately without bothering human users.\n\n## Conclusion\n\nEffective bot management requires looking at the holistic picture. You analyze the IP reputation, the ASN, the behavioral patterns, the reverse DNS records, and the HTTP headers collectively. IP Shield aggregates these vast, complex signals into a unified threat intelligence layer.\n\nThis unified approach allows you to set dynamic, highly granular thresholds. You block requests with a definitive spoofing flag outright, challenge suspicious gray-area requests with invisible Proof of Work puzzles, and allow verified SEO traffic seamless, accelerated access. \n\nThe evolution of bot management has moved definitively away from static string-matching rules. To protect modern infrastructure, engineering teams must embrace dynamic, context-aware intelligence networks.\n","40 min",{"title":137,"description":1899},[1908,1909,1910,1911,1912],"engineering","bot-management","seo","architecture","web-scraping","kvjQkw6JxRHjk5WKFXl4xVLb9a-yS8apUi25NFIEkCY",{"id":212,"title":137,"author":213,"authorGithub":214,"authorGithubUserName":215,"authorImg":216,"body":1915,"date":1898,"description":1899,"extension":1900,"featured":61,"icon":1901,"image":1902,"meta":3161,"navigation":10,"path":138,"rawbody":1904,"readingTime":1905,"seo":3162,"stem":139,"tags":3163,"__hash__":1913},{"type":218,"value":1916,"toc":3143},[1917,1919,1921,1931,1933,1937,1939,1941,1943,1953,1955,1959,1961,1965,1967,1969,2037,2041,2043,2045,2051,2053,2055,2057,2087,2089,2091,2093,2095,2097,2099,2863,2867,2869,2871,3067,3069,3105,3107,3109,3113,3115,3117,3133,3135,3137,3139,3141],[221,1918,137],{"id":223},[225,1920,227],{},[225,1922,230,1923,235,1925,239,1927,243,1929,247],{},[232,1924,234],{},[232,1926,238],{},[232,1928,242],{},[232,1930,246],{},[225,1932,250],{},[252,1934,1935],{},[225,1936,256],{},[258,1938,261],{"id":260},[225,1940,264],{},[266,1942,269],{"id":268},[225,1944,272,1945,239,1947,279,1949,283,1951,286],{},[232,1946,275],{},[232,1948,278],{},[232,1950,282],{},[232,1952,238],{},[266,1954,290],{"id":289},[225,1956,293,1957,297],{},[232,1958,296],{},[266,1960,301],{"id":300},[225,1962,304,1963,308],{},[232,1964,307],{},[266,1966,312],{"id":311},[225,1968,315],{},[317,1970,1971],{"className":319,"code":320,"language":321,"meta":322,"style":322},[232,1972,1973,1977,1981,1985,1989,1993,1997,2001,2005,2009,2013,2017,2021,2025,2029,2033],{"__ignoreMap":322},[326,1974,1975],{"class":328,"line":329},[326,1976,332],{},[326,1978,1979],{"class":328,"line":335},[326,1980,338],{},[326,1982,1983],{"class":328,"line":341},[326,1984,344],{},[326,1986,1987],{"class":328,"line":347},[326,1988,350],{},[326,1990,1991],{"class":328,"line":353},[326,1992,356],{},[326,1994,1995],{"class":328,"line":359},[326,1996,362],{"emptyLinePlaceholder":10},[326,1998,1999],{"class":328,"line":365},[326,2000,368],{},[326,2002,2003],{"class":328,"line":371},[326,2004,374],{},[326,2006,2007],{"class":328,"line":377},[326,2008,380],{},[326,2010,2011],{"class":328,"line":383},[326,2012,386],{},[326,2014,2015],{"class":328,"line":389},[326,2016,392],{},[326,2018,2019],{"class":328,"line":395},[326,2020,398],{},[326,2022,2023],{"class":328,"line":401},[326,2024,404],{},[326,2026,2027],{"class":328,"line":407},[326,2028,410],{},[326,2030,2031],{"class":328,"line":413},[326,2032,416],{},[326,2034,2035],{"class":328,"line":419},[326,2036,422],{},[424,2038,2039],{},[225,2040,428],{},[258,2042,432],{"id":431},[225,2044,435],{},[225,2046,438,2047,442,2049,446],{},[232,2048,441],{},[232,2050,445],{},[225,2052,449],{},[266,2054,453],{"id":452},[225,2056,456],{},[458,2058,2059,2067,2075,2081],{},[461,2060,2061,467,2063,471,2065,446],{},[464,2062,466],{},[232,2064,470],{},[232,2066,474],{},[461,2068,2069,480,2071,283,2073,446],{},[464,2070,479],{},[232,2072,483],{},[232,2074,486],{},[461,2076,2077,492,2079,495],{},[464,2078,491],{},[232,2080,474],{},[461,2082,2083,501,2085,504],{},[464,2084,500],{},[232,2086,470],{},[225,2088,507],{},[266,2090,511],{"id":510},[225,2092,514],{},[225,2094,517],{},[258,2096,521],{"id":520},[225,2098,524],{},[317,2100,2101],{"className":527,"code":528,"filename":529,"language":530,"meta":322,"style":322},[232,2102,2103,2123,2151,2155,2159,2185,2189,2193,2197,2215,2219,2241,2245,2275,2305,2321,2325,2329,2397,2403,2407,2411,2415,2433,2439,2443,2447,2453,2457,2487,2491,2495,2517,2547,2551,2555,2569,2575,2589,2601,2605,2609,2613,2617,2631,2635,2649,2653,2657,2683,2687,2695,2699,2703,2707,2711,2735,2749,2755,2769,2781,2785,2789,2793,2797,2801,2807,2811,2823,2827,2849,2855,2859],{"__ignoreMap":322},[326,2104,2105,2107,2109,2111,2113,2115,2117,2119,2121],{"class":328,"line":329},[326,2106,538],{"class":537},[326,2108,542],{"class":541},[326,2110,546],{"class":545},[326,2112,549],{"class":541},[326,2114,552],{"class":537},[326,2116,556],{"class":555},[326,2118,560],{"class":559},[326,2120,563],{"class":555},[326,2122,566],{"class":541},[326,2124,2125,2127,2129,2131,2133,2135,2137,2139,2141,2143,2145,2147,2149],{"class":328,"line":335},[326,2126,538],{"class":537},[326,2128,542],{"class":541},[326,2130,575],{"class":545},[326,2132,239],{"class":541},[326,2134,580],{"class":545},[326,2136,239],{"class":541},[326,2138,585],{"class":545},[326,2140,549],{"class":541},[326,2142,552],{"class":537},[326,2144,556],{"class":555},[326,2146,266],{"class":559},[326,2148,563],{"class":555},[326,2150,566],{"class":541},[326,2152,2153],{"class":328,"line":341},[326,2154,362],{"emptyLinePlaceholder":10},[326,2156,2157],{"class":328,"line":347},[326,2158,607],{"class":606},[326,2160,2161,2163,2165,2167,2169,2171,2173,2175,2177,2179,2181,2183],{"class":328,"line":353},[326,2162,613],{"class":612},[326,2164,617],{"class":616},[326,2166,621],{"class":620},[326,2168,625],{"class":624},[326,2170,629],{"class":628},[326,2172,632],{"class":541},[326,2174,635],{"class":545},[326,2176,446],{"class":541},[326,2178,640],{"class":545},[326,2180,446],{"class":541},[326,2182,646],{"class":645},[326,2184,649],{"class":541},[326,2186,2187],{"class":328,"line":359},[326,2188,362],{"emptyLinePlaceholder":10},[326,2190,2191],{"class":328,"line":365},[326,2192,658],{"class":606},[326,2194,2195],{"class":328,"line":371},[326,2196,663],{"class":606},[326,2198,2199,2201,2203,2205,2207,2209,2211,2213],{"class":328,"line":377},[326,2200,613],{"class":612},[326,2202,670],{"class":616},[326,2204,621],{"class":620},[326,2206,625],{"class":624},[326,2208,677],{"class":628},[326,2210,680],{"class":541},[326,2212,684],{"class":683},[326,2214,687],{"class":541},[326,2216,2217],{"class":328,"line":383},[326,2218,362],{"emptyLinePlaceholder":10},[326,2220,2221,2223,2225,2227,2229,2231,2233,2235,2237,2239],{"class":328,"line":389},[326,2222,696],{"class":537},[326,2224,699],{"class":537},[326,2226,702],{"class":628},[326,2228,632],{"class":541},[326,2230,707],{"class":612},[326,2232,710],{"class":541},[326,2234,714],{"class":713},[326,2236,717],{"class":541},[326,2238,720],{"class":612},[326,2240,723],{"class":541},[326,2242,2243],{"class":328,"line":395},[326,2244,728],{"class":606},[326,2246,2247,2249,2251,2253,2255,2257,2259,2261,2263,2265,2267,2269,2271,2273],{"class":328,"line":401},[326,2248,733],{"class":612},[326,2250,736],{"class":616},[326,2252,621],{"class":620},[326,2254,741],{"class":628},[326,2256,632],{"class":541},[326,2258,714],{"class":545},[326,2260,748],{"class":541},[326,2262,751],{"class":545},[326,2264,755],{"class":754},[326,2266,759],{"class":758},[326,2268,762],{"class":541},[326,2270,765],{"class":620},[326,2272,768],{"class":555},[326,2274,566],{"class":541},[326,2276,2277,2279,2281,2283,2285,2287,2289,2291,2293,2295,2297,2299,2301,2303],{"class":328,"line":407},[326,2278,733],{"class":612},[326,2280,777],{"class":616},[326,2282,621],{"class":620},[326,2284,782],{"class":628},[326,2286,632],{"class":541},[326,2288,714],{"class":545},[326,2290,239],{"class":541},[326,2292,563],{"class":555},[326,2294,793],{"class":559},[326,2296,563],{"class":555},[326,2298,717],{"class":541},[326,2300,765],{"class":620},[326,2302,768],{"class":555},[326,2304,566],{"class":541},[326,2306,2307,2309,2311,2313,2315,2317,2319],{"class":328,"line":413},[326,2308,733],{"class":612},[326,2310,810],{"class":616},[326,2312,621],{"class":620},[326,2314,815],{"class":545},[326,2316,446],{"class":541},[326,2318,820],{"class":545},[326,2320,566],{"class":541},[326,2322,2323],{"class":328,"line":419},[326,2324,827],{"class":541},[326,2326,2327],{"class":328,"line":830},[326,2328,833],{"class":606},[326,2330,2331,2333,2335,2337,2339,2341,2343,2345,2347,2349,2351,2353,2355,2357,2359,2361,2363,2365,2367,2369,2371,2373,2375,2377,2379,2381,2383,2385,2387,2389,2391,2393,2395],{"class":328,"line":836},[326,2332,839],{"class":537},[326,2334,710],{"class":541},[326,2336,820],{"class":545},[326,2338,446],{"class":541},[326,2340,848],{"class":628},[326,2342,632],{"class":541},[326,2344,563],{"class":555},[326,2346,855],{"class":559},[326,2348,563],{"class":555},[326,2350,717],{"class":541},[326,2352,765],{"class":620},[326,2354,810],{"class":545},[326,2356,446],{"class":541},[326,2358,868],{"class":628},[326,2360,632],{"class":541},[326,2362,874],{"class":873},[326,2364,878],{"class":877},[326,2366,632],{"class":881},[326,2368,885],{"class":884},[326,2370,889],{"class":888},[326,2372,892],{"class":884},[326,2374,889],{"class":888},[326,2376,897],{"class":884},[326,2378,889],{"class":888},[326,2380,902],{"class":884},[326,2382,889],{"class":888},[326,2384,907],{"class":884},[326,2386,889],{"class":888},[326,2388,912],{"class":884},[326,2390,915],{"class":881},[326,2392,918],{"class":888},[326,2394,874],{"class":873},[326,2396,923],{"class":541},[326,2398,2399,2401],{"class":328,"line":926},[326,2400,929],{"class":537},[326,2402,566],{"class":541},[326,2404,2405],{"class":328,"line":934},[326,2406,937],{"class":541},[326,2408,2409],{"class":328,"line":940},[326,2410,362],{"emptyLinePlaceholder":10},[326,2412,2413],{"class":328,"line":945},[326,2414,948],{"class":606},[326,2416,2417,2419,2421,2423,2425,2427,2429,2431],{"class":328,"line":951},[326,2418,839],{"class":537},[326,2420,710],{"class":541},[326,2422,958],{"class":545},[326,2424,446],{"class":541},[326,2426,963],{"class":628},[326,2428,632],{"class":541},[326,2430,968],{"class":545},[326,2432,923],{"class":541},[326,2434,2435,2437],{"class":328,"line":973},[326,2436,929],{"class":537},[326,2438,566],{"class":541},[326,2440,2441],{"class":328,"line":980},[326,2442,937],{"class":541},[326,2444,2445],{"class":328,"line":985},[326,2446,827],{"class":541},[326,2448,2449,2451],{"class":328,"line":990},[326,2450,993],{"class":537},[326,2452,723],{"class":541},[326,2454,2455],{"class":328,"line":998},[326,2456,1001],{"class":606},[326,2458,2459,2461,2463,2465,2467,2469,2471,2473,2475,2477,2479,2481,2483,2485],{"class":328,"line":1004},[326,2460,1007],{"class":612},[326,2462,1010],{"class":616},[326,2464,621],{"class":620},[326,2466,1015],{"class":537},[326,2468,617],{"class":545},[326,2470,446],{"class":541},[326,2472,1022],{"class":545},[326,2474,446],{"class":541},[326,2476,1027],{"class":628},[326,2478,1030],{"class":541},[326,2480,968],{"class":545},[326,2482,239],{"class":541},[326,2484,1037],{"class":545},[326,2486,1040],{"class":541},[326,2488,2489],{"class":328,"line":1043},[326,2490,1046],{"class":541},[326,2492,2493],{"class":328,"line":1049},[326,2494,1052],{"class":606},[326,2496,2497,2499,2501,2503,2505,2507,2509,2511,2513,2515],{"class":328,"line":1055},[326,2498,1058],{"class":537},[326,2500,710],{"class":541},[326,2502,1063],{"class":545},[326,2504,446],{"class":541},[326,2506,1068],{"class":545},[326,2508,1071],{"class":620},[326,2510,1010],{"class":545},[326,2512,446],{"class":541},[326,2514,1078],{"class":545},[326,2516,1081],{"class":541},[326,2518,2519,2521,2523,2525,2527,2529,2531,2533,2535,2537,2539,2541,2543,2545],{"class":328,"line":1084},[326,2520,1087],{"class":545},[326,2522,446],{"class":541},[326,2524,1092],{"class":628},[326,2526,632],{"class":541},[326,2528,1097],{"class":559},[326,2530,1100],{"class":612},[326,2532,968],{"class":545},[326,2534,1105],{"class":612},[326,2536,1108],{"class":559},[326,2538,1100],{"class":612},[326,2540,1037],{"class":545},[326,2542,1105],{"class":612},[326,2544,1117],{"class":559},[326,2546,649],{"class":541},[326,2548,2549],{"class":328,"line":1122},[326,2550,1125],{"class":541},[326,2552,2553],{"class":328,"line":1128},[326,2554,1131],{"class":606},[326,2556,2557,2559,2561,2563,2565,2567],{"class":328,"line":1134},[326,2558,1137],{"class":628},[326,2560,632],{"class":541},[326,2562,714],{"class":545},[326,2564,239],{"class":541},[326,2566,1147],{"class":1146},[326,2568,649],{"class":541},[326,2570,2571,2573],{"class":328,"line":1152},[326,2572,1155],{"class":537},[326,2574,1158],{"class":541},[326,2576,2577,2579,2581,2583,2585,2587],{"class":328,"line":1161},[326,2578,1164],{"class":545},[326,2580,755],{"class":754},[326,2582,556],{"class":555},[326,2584,1171],{"class":559},[326,2586,563],{"class":555},[326,2588,1176],{"class":541},[326,2590,2591,2593,2595,2597,2599],{"class":328,"line":1179},[326,2592,1182],{"class":545},[326,2594,755],{"class":754},[326,2596,556],{"class":555},[326,2598,1189],{"class":559},[326,2600,1192],{"class":555},[326,2602,2603],{"class":328,"line":1195},[326,2604,1198],{"class":541},[326,2606,2607],{"class":328,"line":1201},[326,2608,1204],{"class":541},[326,2610,2611],{"class":328,"line":1207},[326,2612,1046],{"class":541},[326,2614,2615],{"class":328,"line":1212},[326,2616,1215],{"class":606},[326,2618,2619,2621,2623,2625,2627,2629],{"class":328,"line":1218},[326,2620,1058],{"class":537},[326,2622,710],{"class":541},[326,2624,1063],{"class":545},[326,2626,446],{"class":541},[326,2628,1229],{"class":545},[326,2630,1081],{"class":541},[326,2632,2633],{"class":328,"line":1234},[326,2634,1237],{"class":606},[326,2636,2637,2639,2641,2643,2645,2647],{"class":328,"line":1240},[326,2638,1243],{"class":545},[326,2640,446],{"class":541},[326,2642,1248],{"class":628},[326,2644,632],{"class":541},[326,2646,968],{"class":545},[326,2648,649],{"class":541},[326,2650,2651],{"class":328,"line":1257},[326,2652,1125],{"class":541},[326,2654,2655],{"class":328,"line":1262},[326,2656,1265],{"class":606},[326,2658,2659,2661,2663,2665,2667,2669,2671,2673,2675,2677,2679,2681],{"class":328,"line":1268},[326,2660,1271],{"class":537},[326,2662,710],{"class":541},[326,2664,958],{"class":545},[326,2666,446],{"class":541},[326,2668,1280],{"class":545},[326,2670,1283],{"class":620},[326,2672,1286],{"class":1146},[326,2674,717],{"class":541},[326,2676,958],{"class":545},[326,2678,446],{"class":541},[326,2680,1295],{"class":628},[326,2682,1298],{"class":541},[326,2684,2685],{"class":328,"line":1301},[326,2686,1125],{"class":541},[326,2688,2689,2691,2693],{"class":328,"line":1306},[326,2690,1155],{"class":537},[326,2692,1311],{"class":541},[326,2694,1314],{"class":606},[326,2696,2697],{"class":328,"line":1317},[326,2698,1204],{"class":541},[326,2700,2701],{"class":328,"line":1322},[326,2702,1046],{"class":541},[326,2704,2705],{"class":328,"line":1327},[326,2706,1330],{"class":606},[326,2708,2709],{"class":328,"line":1333},[326,2710,1336],{"class":606},[326,2712,2713,2715,2717,2719,2721,2723,2725,2727,2729,2731,2733],{"class":328,"line":1339},[326,2714,1058],{"class":537},[326,2716,710],{"class":541},[326,2718,1063],{"class":545},[326,2720,446],{"class":541},[326,2722,1068],{"class":545},[326,2724,1071],{"class":620},[326,2726,1354],{"class":620},[326,2728,1063],{"class":545},[326,2730,446],{"class":541},[326,2732,1229],{"class":545},[326,2734,1081],{"class":541},[326,2736,2737,2739,2741,2743,2745,2747],{"class":328,"line":1365},[326,2738,1137],{"class":628},[326,2740,632],{"class":541},[326,2742,714],{"class":545},[326,2744,239],{"class":541},[326,2746,1376],{"class":1146},[326,2748,649],{"class":541},[326,2750,2751,2753],{"class":328,"line":1381},[326,2752,1155],{"class":537},[326,2754,1158],{"class":541},[326,2756,2757,2759,2761,2763,2765,2767],{"class":328,"line":1388},[326,2758,1164],{"class":545},[326,2760,755],{"class":754},[326,2762,556],{"class":555},[326,2764,1397],{"class":559},[326,2766,563],{"class":555},[326,2768,1176],{"class":541},[326,2770,2771,2773,2775,2777,2779],{"class":328,"line":1404},[326,2772,1182],{"class":545},[326,2774,755],{"class":754},[326,2776,556],{"class":555},[326,2778,1413],{"class":559},[326,2780,1192],{"class":555},[326,2782,2783],{"class":328,"line":1418},[326,2784,1198],{"class":541},[326,2786,2787],{"class":328,"line":1423},[326,2788,1204],{"class":541},[326,2790,2791],{"class":328,"line":1428},[326,2792,362],{"emptyLinePlaceholder":10},[326,2794,2795],{"class":328,"line":1433},[326,2796,1436],{"class":606},[326,2798,2799],{"class":328,"line":1439},[326,2800,1442],{"class":606},[326,2802,2803,2805],{"class":328,"line":1445},[326,2804,929],{"class":537},[326,2806,566],{"class":541},[326,2808,2809],{"class":328,"line":1452},[326,2810,1046],{"class":541},[326,2812,2813,2815,2817,2819,2821],{"class":328,"line":1457},[326,2814,1460],{"class":541},[326,2816,1463],{"class":537},[326,2818,710],{"class":541},[326,2820,1468],{"class":545},[326,2822,1081],{"class":541},[326,2824,2825],{"class":328,"line":1473},[326,2826,1476],{"class":606},[326,2828,2829,2831,2833,2835,2837,2839,2841,2843,2845,2847],{"class":328,"line":1479},[326,2830,1482],{"class":545},[326,2832,446],{"class":541},[326,2834,1487],{"class":628},[326,2836,632],{"class":541},[326,2838,563],{"class":555},[326,2840,1494],{"class":559},[326,2842,563],{"class":555},[326,2844,239],{"class":541},[326,2846,1468],{"class":545},[326,2848,649],{"class":541},[326,2850,2851,2853],{"class":328,"line":1505},[326,2852,929],{"class":537},[326,2854,566],{"class":541},[326,2856,2857],{"class":328,"line":1512},[326,2858,937],{"class":541},[326,2860,2861],{"class":328,"line":1517},[326,2862,1520],{"class":541},[1522,2864,2865],{},[225,2866,1526],{},[258,2868,1530],{"id":1529},[225,2870,1533],{},[317,2872,2873],{"className":1536,"code":1537,"filename":1538,"language":1539,"meta":322,"style":322},[232,2874,2875,2879,2883,2887,2891,2895,2899,2903,2907,2911,2915,2919,2923,2927,2931,2935,2939,2943,2947,2951,2955,2959,2963,2967,2971,2975,2979,2983,2987,2991,2995,2999,3003,3007,3011,3015,3019,3023,3027,3031,3035,3039,3043,3047,3051,3055,3059,3063],{"__ignoreMap":322},[326,2876,2877],{"class":328,"line":329},[326,2878,1546],{},[326,2880,2881],{"class":328,"line":335},[326,2882,1551],{},[326,2884,2885],{"class":328,"line":341},[326,2886,1556],{},[326,2888,2889],{"class":328,"line":347},[326,2890,362],{"emptyLinePlaceholder":10},[326,2892,2893],{"class":328,"line":353},[326,2894,1565],{},[326,2896,2897],{"class":328,"line":359},[326,2898,362],{"emptyLinePlaceholder":10},[326,2900,2901],{"class":328,"line":365},[326,2902,1574],{},[326,2904,2905],{"class":328,"line":371},[326,2906,1579],{},[326,2908,2909],{"class":328,"line":377},[326,2910,1584],{},[326,2912,2913],{"class":328,"line":383},[326,2914,1579],{},[326,2916,2917],{"class":328,"line":389},[326,2918,1593],{},[326,2920,2921],{"class":328,"line":395},[326,2922,1598],{},[326,2924,2925],{"class":328,"line":401},[326,2926,1046],{},[326,2928,2929],{"class":328,"line":407},[326,2930,1607],{},[326,2932,2933],{"class":328,"line":413},[326,2934,1612],{},[326,2936,2937],{"class":328,"line":419},[326,2938,1617],{},[326,2940,2941],{"class":328,"line":830},[326,2942,1622],{},[326,2944,2945],{"class":328,"line":836},[326,2946,362],{"emptyLinePlaceholder":10},[326,2948,2949],{"class":328,"line":926},[326,2950,1631],{},[326,2952,2953],{"class":328,"line":934},[326,2954,1636],{},[326,2956,2957],{"class":328,"line":940},[326,2958,1641],{},[326,2960,2961],{"class":328,"line":945},[326,2962,1646],{},[326,2964,2965],{"class":328,"line":951},[326,2966,1651],{},[326,2968,2969],{"class":328,"line":973},[326,2970,1656],{},[326,2972,2973],{"class":328,"line":980},[326,2974,1661],{},[326,2976,2977],{"class":328,"line":985},[326,2978,1666],{},[326,2980,2981],{"class":328,"line":990},[326,2982,1671],{},[326,2984,2985],{"class":328,"line":998},[326,2986,1676],{},[326,2988,2989],{"class":328,"line":1004},[326,2990,1681],{},[326,2992,2993],{"class":328,"line":1043},[326,2994,1686],{},[326,2996,2997],{"class":328,"line":1049},[326,2998,1691],{},[326,3000,3001],{"class":328,"line":1055},[326,3002,1696],{},[326,3004,3005],{"class":328,"line":1084},[326,3006,1701],{},[326,3008,3009],{"class":328,"line":1122},[326,3010,1706],{},[326,3012,3013],{"class":328,"line":1128},[326,3014,1711],{},[326,3016,3017],{"class":328,"line":1134},[326,3018,1716],{},[326,3020,3021],{"class":328,"line":1152},[326,3022,1721],{},[326,3024,3025],{"class":328,"line":1161},[326,3026,1696],{},[326,3028,3029],{"class":328,"line":1179},[326,3030,1730],{},[326,3032,3033],{"class":328,"line":1195},[326,3034,1735],{},[326,3036,3037],{"class":328,"line":1201},[326,3038,1711],{},[326,3040,3041],{"class":328,"line":1207},[326,3042,1716],{},[326,3044,3045],{"class":328,"line":1212},[326,3046,1748],{},[326,3048,3049],{"class":328,"line":1218},[326,3050,1753],{},[326,3052,3053],{"class":328,"line":1234},[326,3054,1758],{},[326,3056,3057],{"class":328,"line":1240},[326,3058,1763],{},[326,3060,3061],{"class":328,"line":1257},[326,3062,1671],{},[326,3064,3065],{"class":328,"line":1262},[326,3066,1772],{},[225,3068,1775],{},[317,3070,3071],{"className":1536,"code":1778,"filename":1779,"language":1539,"meta":322,"style":322},[232,3072,3073,3077,3081,3085,3089,3093,3097,3101],{"__ignoreMap":322},[326,3074,3075],{"class":328,"line":329},[326,3076,1786],{},[326,3078,3079],{"class":328,"line":335},[326,3080,1791],{},[326,3082,3083],{"class":328,"line":341},[326,3084,362],{"emptyLinePlaceholder":10},[326,3086,3087],{"class":328,"line":347},[326,3088,1800],{},[326,3090,3091],{"class":328,"line":353},[326,3092,362],{"emptyLinePlaceholder":10},[326,3094,3095],{"class":328,"line":359},[326,3096,1809],{},[326,3098,3099],{"class":328,"line":365},[326,3100,1814],{},[326,3102,3103],{"class":328,"line":371},[326,3104,1819],{},[258,3106,1823],{"id":1822},[225,3108,1826],{},[225,3110,1829,3111,446],{},[464,3112,1832],{},[225,3114,1835],{},[266,3116,1839],{"id":1838},[458,3118,3119,3123,3127],{},[461,3120,3121,1847],{},[464,3122,1846],{},[461,3124,3125,1853],{},[464,3126,1852],{},[461,3128,3129,1859,3131,1862],{},[464,3130,1858],{},[232,3132,275],{},[258,3134,1866],{"id":1865},[225,3136,1869],{},[225,3138,1872],{},[225,3140,1875],{},[1877,3142,1879],{},{"title":322,"searchDepth":335,"depth":335,"links":3144},[3145,3151,3155,3156,3157,3160],{"id":260,"depth":335,"text":261,"children":3146},[3147,3148,3149,3150],{"id":268,"depth":341,"text":269},{"id":289,"depth":341,"text":290},{"id":300,"depth":341,"text":301},{"id":311,"depth":341,"text":312},{"id":431,"depth":335,"text":432,"children":3152},[3153,3154],{"id":452,"depth":341,"text":453},{"id":510,"depth":341,"text":511},{"id":520,"depth":335,"text":521},{"id":1529,"depth":335,"text":1530},{"id":1822,"depth":335,"text":1823,"children":3158},[3159],{"id":1838,"depth":341,"text":1839},{"id":1865,"depth":335,"text":1866},{},{"title":137,"description":1899},[1908,1909,1910,1911,1912],1782044516185]