{ "generated": "2026-10-19", "method": "robots.txt resolved per agent: longest matching user-agent group wins, else wildcard. blocked = Disallow: / with no Allow carve-out. partial = some paths disallowed. llms_txt counted only when the body is plain text.", "agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "sites": [ { "domain": "vercel.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 85, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing" ], "server_leaks": [ "x-powered-by: Next.js, Payload" ] }, { "domain": "stackoverflow.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 81, "headers_missing": [ "strict-transport-security:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "github.com", "robots_txt": true, "blocked": [ "Bytespider" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "A", "headers_score": 93, "headers_missing": [ "permissions-policy:missing" ], "server_leaks": [ "server: github.com" ] }, { "domain": "cloudflare.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "B", "headers_score": 87, "headers_missing": [ "content-security-policy:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "stripe.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 89, "headers_missing": [ "referrer-policy:weak", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "supabase.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "resend.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "B", "headers_score": 89, "headers_missing": [ "content-security-policy:weak" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "neon.tech", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "sendgrid.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "F", "headers_score": 29, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak" ], "server_leaks": [ "server: AmazonS3", "via: 1.1 e3a402c7b046c2ff456c30009b54d8b6.cloudfront.net (CloudFront)" ] }, { "domain": "planetscale.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "C", "headers_score": 55, "headers_missing": [ "strict-transport-security:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak" ], "server_leaks": [] }, { "domain": "twilio.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 66, "headers_missing": [ "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak" ], "server_leaks": [ "via: 1.1 e3a402c7b046c2ff456c30009b54d8b6.cloudfront.net (CloudFront)" ] }, { "domain": "netlify.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "C", "headers_score": 56, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing" ], "server_leaks": [] }, { "domain": "fly.io", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 74, "headers_missing": [ "strict-transport-security:missing", "permissions-policy:missing", "set-cookie:weak" ], "server_leaks": [ "server: Fly/42494bfe9c (2026-10-15)", "via: 1.1 fly.io" ] }, { "domain": "redis.io", "robots_txt": true, "blocked": [], "partial": [ "OAI-SearchBot", "ClaudeBot", "Claude-User", "anthropic-ai", "Perplexity-User", "Bytespider", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 77, "headers_missing": [ "strict-transport-security:weak", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "gitlab.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "D", "headers_score": 50, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak" ], "server_leaks": [] }, { "domain": "railway.app", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 64, "headers_missing": [ "strict-transport-security:missing", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: railway-hikari" ] }, { "domain": "render.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 88, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "linear.app", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 85, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing" ], "server_leaks": [ "via: 1.1 google" ] }, { "domain": "heroku.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 89, "headers_missing": [ "content-security-policy:weak", "referrer-policy:weak" ], "server_leaks": [] }, { "domain": "figma.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "Google-Extended", "Bytespider", "CCBot", "cohere-ai" ], "partial": [ "Claude-Web", "anthropic-ai", "Perplexity-User", "Amazonbot", "Applebot-Extended", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 79, "headers_missing": [ "content-security-policy:weak", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "via: 1.1 fe624e5ec67fb7582d7a2f57fd2d7078.cloudfront.net (CloudFront)" ] }, { "domain": "notion.so", "robots_txt": true, "blocked": [ "Amazonbot" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 67, "headers_missing": [ "content-security-policy:weak", "x-content-type-options:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "zoom.us", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 50, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak" ], "server_leaks": [] }, { "domain": "slack.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 61, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [ "via: 1.1 slack-prod.tinyspeck.com, ingress-www-iad-external-prod-0d6cf87ea66d4987c,ingress-edge-pdx-prod-0478ee1826de62f30" ] }, { "domain": "box.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "C", "headers_score": 74, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:weak", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "via: varnish" ] }, { "domain": "digitalocean.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 56, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:weak", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "dropbox.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 80, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: envoy" ] }, { "domain": "asana.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 78, "headers_missing": [ "content-security-policy:weak", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "atlassian.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 71, "headers_missing": [ "content-security-policy:weak", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: AtlassianEdge", "via: 1.1 beb95641b50d24f031b9c1b3427b51c6.cloudfront.net (CloudFront)" ] }, { "domain": "zapier.com", "robots_txt": true, "blocked": [], "partial": [ "Claude-Web", "anthropic-ai", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 85, "headers_missing": [ "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "via: 1.1 260bf478e70912d685a952d11275bab6.cloudfront.net (CloudFront)" ] }, { "domain": "trello.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "D", "headers_score": 49, "headers_missing": [ "content-security-policy:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: AtlassianEdge", "via: 1.1 d585c5a4fc7aafde7a406ed62c00241c.cloudfront.net (CloudFront)" ] }, { "domain": "mongodb.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "F", "headers_score": 31, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: istio-envoy", "x-powered-by: Next.js", "via: 1.1 89fb248da6d030379dec9569a47a0c92.cloudfront.net (CloudFront)" ] }, { "domain": "airtable.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 53, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:weak" ], "server_leaks": [ "x-powered-by: Next.js", "via: 1.1 cf0358efeeb32c222cf57420ce241ce8.cloudfront.net (CloudFront)" ] }, { "domain": "make.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "zendesk.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 77, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:weak", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "monday.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "D", "headers_score": 35, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "via: 1.1 4869e7192f8697c2504b2a54821a5c44.cloudfront.net (CloudFront)" ] }, { "domain": "shopify.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 60, "headers_missing": [ "content-security-policy:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "intercom.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 68, "headers_missing": [ "content-security-policy:weak", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "x-powered-by: Next.js", "via: 1.1 102b84cd60f4fa6e88ba6aa2243458ba.cloudfront.net (CloudFront)" ] }, { "domain": "hubspot.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "A", "headers_score": 93, "headers_missing": [ "referrer-policy:weak", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "salesforce.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 58, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "wordpress.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 66, "headers_missing": [ "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "ghost.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 89, "headers_missing": [ "content-security-policy:weak", "referrer-policy:weak" ], "server_leaks": [] }, { "domain": "squarespace.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "D", "headers_score": 47, "headers_missing": [ "content-security-policy:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: Squarespace" ] }, { "domain": "webflow.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 76, "headers_missing": [ "content-security-policy:weak", "x-content-type-options:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "dev.to", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "A", "headers_score": 90, "headers_missing": [ "permissions-policy:missing" ], "server_leaks": [ "server: Heroku", "via: 1.1 heroku-router, 1.1 varnish, 1.1 varnish" ] }, { "domain": "hashnode.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "D", "headers_score": 49, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "reddit.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [], "llms_txt": "html-soft404", "headers_grade": "B", "headers_score": 81, "headers_missing": [ "referrer-policy:missing", "permissions-policy:missing", "content-type:weak", "set-cookie:weak" ], "server_leaks": [ "server: snooserv", "via: 1.1 varnish" ] }, { "domain": "substack.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 66, "headers_missing": [ "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "x-powered-by: Express" ] }, { "domain": "medium.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "Bytespider", "Amazonbot", "Applebot-Extended", "Meta-ExternalAgent" ], "partial": [ "OAI-SearchBot", "ChatGPT-User", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "cohere-ai", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "B", "headers_score": 76, "headers_missing": [ "content-security-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "x.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "DuckAssistBot", "MistralAI-User" ], "partial": [ "Google-Extended", "Meta-ExternalAgent", "Meta-ExternalFetcher" ], "llms_txt": "no (404)", "headers_grade": "A", "headers_score": 93, "headers_missing": [ "permissions-policy:missing" ], "server_leaks": [ "server: cloudflare envoy" ] }, { "domain": "linkedin.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "html-soft404", "headers_grade": "B", "headers_score": 89, "headers_missing": [ "permissions-policy:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "facebook.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [], "llms_txt": "html-soft404", "headers_grade": "A", "headers_score": 92, "headers_missing": [ "referrer-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "instagram.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [], "llms_txt": "html-soft404", "headers_grade": "A", "headers_score": 92, "headers_missing": [ "referrer-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "pinterest.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "MistralAI-User" ], "partial": [ "Applebot-Extended", "DuckAssistBot" ], "llms_txt": "html-soft404", "headers_grade": "B", "headers_score": 86, "headers_missing": [ "permissions-policy:missing", "cache-control:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "youtube.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "A", "headers_score": 90, "headers_missing": [ "referrer-policy:missing", "set-cookie:weak" ], "server_leaks": [ "server: ESF" ] }, { "domain": "wikipedia.org", "robots_txt": true, "blocked": [ "Meta-ExternalFetcher" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 44, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: ATS/9.2.15" ] }, { "domain": "mailchimp.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "D", "headers_score": 50, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: istio-envoy" ] }, { "domain": "quora.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "Google-Extended", "Amazonbot" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 81, "headers_missing": [ "strict-transport-security:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "tiktok.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Applebot-Extended", "Meta-ExternalAgent", "DuckAssistBot", "MistralAI-User" ], "partial": [ "Claude-Web", "Amazonbot", "cohere-ai", "Meta-ExternalFetcher" ], "llms_txt": "html-soft404", "headers_grade": "B", "headers_score": 83, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "x-powered-by: Goofy Node" ] }, { "domain": "washingtonpost.com", "robots_txt": true, "blocked": [ "ClaudeBot", "Claude-Web", "anthropic-ai", "PerplexityBot", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "Meta-ExternalAgent", "Meta-ExternalFetcher" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Claude-User", "Google-Extended", "Perplexity-User", "cohere-ai", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "error", "headers_grade": null, "headers_score": null, "headers_missing": [], "server_leaks": [] }, { "domain": "nytimes.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot" ], "partial": [ "Amazonbot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 60, "headers_missing": [ "content-security-policy:weak", "x-content-type-options:missing", "referrer-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: DataDome" ] }, { "domain": "wix.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "D", "headers_score": 47, "headers_missing": [ "content-security-policy:missing", "x-frame-options:missing", "referrer-policy:weak", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: Pepyaka", "via: 1.1 google" ] }, { "domain": "theguardian.com", "robots_txt": true, "blocked": [ "ClaudeBot", "Claude-User", "anthropic-ai", "PerplexityBot", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "Meta-ExternalAgent", "DuckAssistBot" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Claude-Web", "Google-Extended", "Perplexity-User", "cohere-ai", "Meta-ExternalFetcher", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 74, "headers_missing": [ "content-security-policy:weak", "referrer-policy:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "bbc.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent" ], "partial": [ "Claude-User", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 67, "headers_missing": [ "content-security-policy:missing", "referrer-policy:weak", "permissions-policy:missing" ], "server_leaks": [ "server: BBC-GTM", "via: 1.1 BBC-GTM, 1.1 Belfrage, 1.1 varnish" ] }, { "domain": "reuters.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "OAI-SearchBot", "ChatGPT-User" ], "llms_txt": "no (401)", "headers_grade": "C", "headers_score": 69, "headers_missing": [ "strict-transport-security:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: CloudFront", "via: 1.1 95e030d195f2fd02b9dd1216a944c770.cloudfront.net (CloudFront)" ] }, { "domain": "theverge.com", "robots_txt": true, "blocked": [ "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "OAI-SearchBot" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 71, "headers_missing": [ "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "x-powered-by: Next.js", "via: 1.1 varnish, 1.1 varnish" ] }, { "domain": "cnn.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "DuckAssistBot", "MistralAI-User" ], "partial": [ "Google-Extended", "Meta-ExternalAgent", "Meta-ExternalFetcher" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 64, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:weak", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "via: 1.1 varnish, 1.1 varnish" ] }, { "domain": "bloomberg.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot" ], "partial": [ "Perplexity-User", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "D", "headers_score": 47, "headers_missing": [ "content-security-policy:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: Varnish" ] }, { "domain": "wsj.com", "robots_txt": true, "blocked": [ "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Applebot-Extended" ], "llms_txt": "no (403)", "headers_grade": "F", "headers_score": 27, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak" ], "server_leaks": [ "server: CloudFront", "via: 1.1 1d05b7ddc539940764dae5665a215c4e.cloudfront.net (CloudFront)" ] }, { "domain": "wired.com", "robots_txt": true, "blocked": [ "ClaudeBot", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "Claude-Web", "anthropic-ai" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 70, "headers_missing": [ "strict-transport-security:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "via: 1.1 160dd28918b921344551db8cec46dc1a.cloudfront.net (CloudFront)" ] }, { "domain": "techcrunch.com", "robots_txt": true, "blocked": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "anthropic-ai", "Google-Extended", "Bytespider", "CCBot", "Applebot-Extended", "cohere-ai" ], "partial": [ "OAI-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Amazonbot", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 81, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:weak", "referrer-policy:weak" ], "server_leaks": [ "x-powered-by: WordPress VIP " ] }, { "domain": "openai.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "B", "headers_score": 76, "headers_missing": [ "content-security-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "ycombinator.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 76, "headers_missing": [ "content-security-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "producthunt.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 65, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "engadget.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "D", "headers_score": 41, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: nginx/1.18.0 (Ubuntu)", "via: 1.1 07d9077f3449135d04c033a5467c80bc.cloudfront.net (CloudFront)" ] }, { "domain": "google.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 39, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: gws" ] }, { "domain": "anthropic.com", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 65, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:weak", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "arstechnica.com", "robots_txt": true, "blocked": [ "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 77, "headers_missing": [ "strict-transport-security:weak", "content-security-policy:weak", "referrer-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "mozilla.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 82, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing" ], "server_leaks": [ "server: granian", "via: 1.1 google, 1.1 varnish, 1.1 varnish" ] }, { "domain": "apple.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 77, "headers_missing": [ "content-security-policy:weak", "referrer-policy:weak", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: Apple" ] }, { "domain": "indiehackers.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "anthropic-ai", "Google-Extended", "PerplexityBot", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "DuckAssistBot" ], "partial": [ "ChatGPT-User", "Claude-Web", "Claude-User", "Perplexity-User", "Meta-ExternalFetcher", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "B", "headers_score": 81, "headers_missing": [ "strict-transport-security:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "npmjs.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (403)", "headers_grade": "A", "headers_score": 96, "headers_missing": [ "cache-control:missing" ], "server_leaks": [] }, { "domain": "microsoft.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "rubygems.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "A", "headers_score": 90, "headers_missing": [ "permissions-policy:missing" ], "server_leaks": [ "server: RubyGems.org", "via: 1.1 varnish" ] }, { "domain": "crates.io", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 60, "headers_missing": [ "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [ "server: Heroku", "via: 1.1 heroku-router, 1.1 varnish" ] }, { "domain": "go.dev", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 75, "headers_missing": [ "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [ "server: Google Frontend" ] }, { "domain": "docker.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 77, "headers_missing": [ "content-security-policy:missing" ], "server_leaks": [ "via: 1.1 varnish, 1.1 varnish, 1.1 varnish, 1.1 varnish" ] }, { "domain": "pypi.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "A", "headers_score": 93, "headers_missing": [ "cache-control:missing" ], "server_leaks": [ "server: gunicorn" ] }, { "domain": "python.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 55, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing" ], "server_leaks": [ "via: 1.1 varnish, 1.1 varnish, 1.1 varnish" ] }, { "domain": "rust-lang.org", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "F", "headers_score": 31, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: GitHub.com", "via: 1.1 varnish" ] }, { "domain": "react.dev", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "nodejs.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 57, "headers_missing": [ "content-security-policy:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "vuejs.org", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "B", "headers_score": 82, "headers_missing": [ "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "svelte.dev", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "linode.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 80, "headers_missing": [ "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "nextjs.org", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "yes", "headers_grade": "B", "headers_score": 84, "headers_missing": [ "content-security-policy:weak", "permissions-policy:missing", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "tailwindcss.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 49, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "x-powered-by: Next.js" ] }, { "domain": "udemy.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (403)", "headers_grade": "B", "headers_score": 81, "headers_missing": [ "strict-transport-security:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "astro.build", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 35, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak" ], "server_leaks": [] }, { "domain": "mdn.dev", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "B", "headers_score": 88, "headers_missing": [ "permissions-policy:missing", "content-type:weak" ], "server_leaks": [ "server: Google Frontend", "via: 1.1 google, 1.1 varnish, 1.1 varnish, 1.1 varnish" ] }, { "domain": "freecodecamp.org", "robots_txt": true, "blocked": [], "partial": [], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 85, "headers_missing": [ "content-security-policy:weak", "referrer-policy:weak", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "khanacademy.org", "robots_txt": true, "blocked": [ "GPTBot" ], "partial": [ "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "F", "headers_score": 29, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "via: 1.1 varnish" ] }, { "domain": "duolingo.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "F", "headers_score": 26, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: duo-api", "via: 1.1 403407d69be93903419c59c25ec702e2.cloudfront.net (CloudFront)" ] }, { "domain": "coursera.org", "robots_txt": true, "blocked": [ "Meta-ExternalAgent" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "C", "headers_score": 57, "headers_missing": [ "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: envoy", "x-powered-by: Express", "via: 1.1 b7f2e03ea82ba959032d68c9a16e911e.cloudfront.net (CloudFront)" ] }, { "domain": "spotify.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "html-soft404", "headers_grade": "B", "headers_score": 77, "headers_missing": [ "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: envoy", "via: HTTP/1.1 fringe, HTTP/2 edgeproxy, 1.1 google, 1.1 varnish" ] }, { "domain": "amazon.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "anthropic-ai", "Amazonbot", "Applebot-Extended" ], "llms_txt": "no (404)", "headers_grade": "F", "headers_score": 31, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: CloudFront", "via: 1.1 791112ac4f59f6c41c045c01bebdcbfc.cloudfront.net (CloudFront)" ] }, { "domain": "doordash.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (403)", "headers_grade": "A", "headers_score": 96, "headers_missing": [ "cache-control:missing" ], "server_leaks": [] }, { "domain": "w3schools.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "A", "headers_score": 93, "headers_missing": [ "strict-transport-security:weak", "content-type:weak" ], "server_leaks": [] }, { "domain": "uber.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 86, "headers_missing": [ "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "netflix.com", "robots_txt": true, "blocked": [ "Claude-Web", "anthropic-ai", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Google-Extended", "PerplexityBot", "Applebot-Extended" ], "llms_txt": "html-soft404", "headers_grade": "D", "headers_score": 54, "headers_missing": [ "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: envoy", "via: 2 i-02dee69ab1e98dcc4 (us-west-2), 1.1 c000.lax001.ix.nflxvideo.net (FTL-L4-p/6D)" ] }, { "domain": "etsy.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "F", "headers_score": 30, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak" ], "server_leaks": [ "server: DataDome", "via: 1.1 varnish" ] }, { "domain": "airbnb.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "Applebot-Extended" ], "partial": [ "OAI-SearchBot", "ChatGPT-User", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "B", "headers_score": 83, "headers_missing": [ "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "yelp.com", "robots_txt": true, "blocked": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "partial": [], "llms_txt": "no (404)", "headers_grade": "F", "headers_score": 31, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: DataDome", "via: 1.1 varnish" ] }, { "domain": "lyft.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 70, "headers_missing": [ "content-security-policy:missing", "permissions-policy:missing" ], "server_leaks": [ "server: envoy-iad", "via: 1.1 0e307783b98e9dae77cb4a78528644b2.cloudfront.net (CloudFront)" ] }, { "domain": "tripadvisor.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent" ], "partial": [ "OAI-SearchBot", "ChatGPT-User", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "D", "headers_score": 41, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: DataDome", "via: 1.1 d753bdff57c5dcd160113c97de38707a.cloudfront.net (CloudFront)" ] }, { "domain": "expedia.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "B", "headers_score": 80, "headers_missing": [ "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: istio-envoy" ] }, { "domain": "adobe.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "D", "headers_score": 52, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing" ], "server_leaks": [] }, { "domain": "craigslist.org", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "C", "headers_score": 70, "headers_missing": [ "content-security-policy:weak", "x-content-type-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [] }, { "domain": "autodesk.com", "robots_txt": false, "blocked": [], "partial": [], "llms_txt": "no (403)", "headers_grade": "C", "headers_score": 59, "headers_missing": [ "content-security-policy:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: AkamaiGHost" ] }, { "domain": "ebay.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "anthropic-ai", "PerplexityBot", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "Meta-ExternalAgent" ], "partial": [ "OAI-SearchBot", "ChatGPT-User", "Claude-Web", "Claude-User", "Google-Extended", "Perplexity-User", "cohere-ai", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 45, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "content-type:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "server: AkamaiGHost" ] }, { "domain": "booking.com", "robots_txt": true, "blocked": [], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (404)", "headers_grade": "D", "headers_score": 43, "headers_missing": [ "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "cache-control:missing", "content-type:weak" ], "server_leaks": [ "via: 1.1 daa0241023cd2e04dcfd4956fd5471e2.cloudfront.net (CloudFront)" ] }, { "domain": "canva.com", "robots_txt": true, "blocked": [ "GPTBot", "ClaudeBot", "Claude-Web", "anthropic-ai", "Bytespider", "CCBot", "Applebot-Extended", "Meta-ExternalAgent" ], "partial": [ "OAI-SearchBot", "ChatGPT-User", "Claude-User", "Google-Extended", "Amazonbot", "cohere-ai", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "no (403)", "headers_grade": "B", "headers_score": 76, "headers_missing": [ "content-security-policy:missing", "cache-control:missing" ], "server_leaks": [] }, { "domain": "unity.com", "robots_txt": true, "blocked": [ "Bytespider" ], "partial": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-Web", "Claude-User", "anthropic-ai", "Google-Extended", "PerplexityBot", "Perplexity-User", "CCBot", "Amazonbot", "Applebot-Extended", "cohere-ai", "Meta-ExternalAgent", "Meta-ExternalFetcher", "DuckAssistBot", "MistralAI-User" ], "llms_txt": "yes", "headers_grade": "F", "headers_score": 7, "headers_missing": [ "strict-transport-security:missing", "content-security-policy:missing", "x-content-type-options:missing", "x-frame-options:missing", "referrer-policy:missing", "permissions-policy:missing", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak", "set-cookie:weak" ], "server_leaks": [ "x-powered-by: Next.js" ] } ] }