{ "studyId": "ai-crawler-access-study-2026", "sampleFile": "data/ai-crawler-study-2026/sample.json", "sampleNote": "Fixed list of publicly marketed AI products and AI-featured SaaS homepages, chosen before the crawl. Well-known brands and popular AI apps with a public marketing or product homepage. Not a random sample of the web or of all SaaS. notable=true marks widely known brands that prose findings may name; every site is still listed in the results file.", "sampleSelected": "2026-10-14", "runStarted": "2026-10-14T01:25:00.168Z", "runFinished": "2026-10-14T01:27:58.918Z", "userAgent": "AIVisibilityReport-Study/1.0 (+https://ai-biz-site.pages.dev/guides/ai-crawler-access-study-2026; one-time public homepage study; contact: tkdumplingsage@outlook.com)", "timeoutMs": 8000, "pauseMs": 500, "betweenSitesMs": 1000, "requestBudget": "Per site, in order: homepage, /robots.txt, /llms.txt, /sitemap.xml. No retries. No second sitemap URL. If the homepage fetch failed, the other three were not requested.", "checker": "functions/api/check.js runCheck({ sequential: true, skipExtraSitemap: true, includeFacts: true }). The public /api/check path still uses the browser User-Agent and parallel fetches.", "sites": [ { "id": "openai", "name": "OpenAI", "url": "https://openai.com", "category": "Assistant", "notable": true, "checkedAt": "2026-10-14T01:25:00.168Z", "result": { "ok": true, "inputUrl": "https://openai.com/", "finalUrl": "https://openai.com/", "checkedPath": "/", "pageStatus": 403, "truncated": false, "elapsedMs": 1828, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://openai.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://openai.com/ · HTTP 403 · 10117 bytes. JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 6 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 403.", "fix": "Serve /llms.txt as a public plain-text file with HTTP 200." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://openai.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 10117, "words": 6, "h1": "", "title": "", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://openai.com/robots.txt.", "status": 200, "finalUrl": "https://openai.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 98, "sitemapUrls": [ "https://openai.com/sitemap.xml" ], "textLength": 98, "text": "User-agent: *\nAllow: /\nDisallow: /microsoft-for-startups/\n\nSitemap: https://openai.com/sitemap.xml" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://openai.com/llms.txt", "contentType": "text/html; charset=UTF-8", "bytes": 10162, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://openai.com/sitemap.xml", "contentType": "application/xml; charset=utf-8", "bytes": 3851, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "anthropic", "name": "Anthropic", "url": "https://www.anthropic.com", "category": "Assistant", "notable": true, "checkedAt": "2026-10-14T01:25:02.999Z", "result": { "ok": true, "inputUrl": "https://www.anthropic.com/", "finalUrl": "https://www.anthropic.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1980, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.anthropic.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.anthropic.com/ · HTTP 200 · 201364 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "696 words in the raw HTML, ignoring script and style. H1: “AI research and products that put safety at the frontier”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.anthropic.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 201364, "words": 696, "h1": "AI research and products that put safety at the frontier", "title": "Home \\ Anthropic", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.anthropic.com/robots.txt.", "status": 200, "finalUrl": "https://www.anthropic.com/robots.txt", "contentType": "text/plain", "bytes": 71, "sitemapUrls": [ "https://www.anthropic.com/sitemap.xml" ], "textLength": 71, "text": "User-Agent: *\nAllow: /\n\nSitemap: https://www.anthropic.com/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.anthropic.com/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 20610, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.anthropic.com/sitemap.xml", "contentType": "application/xml", "bytes": 71512, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "gemini", "name": "Google Gemini", "url": "https://gemini.google.com", "category": "Assistant", "notable": true, "checkedAt": "2026-10-14T01:25:05.980Z", "result": { "ok": false, "error": "The host did not respond.", "inputUrl": "https://gemini.google.com" } }, { "id": "perplexity", "name": "Perplexity", "url": "https://www.perplexity.ai", "category": "Search", "notable": true, "checkedAt": "2026-10-14T01:25:07.120Z", "result": { "ok": true, "inputUrl": "https://www.perplexity.ai/", "finalUrl": "https://www.perplexity.ai/", "checkedPath": "/", "pageStatus": 403, "truncated": false, "elapsedMs": 1832, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.perplexity.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.perplexity.ai/ · HTTP 403 · 5866 bytes. JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 9 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 403.", "fix": "Serve /llms.txt as a public plain-text file with HTTP 200." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.perplexity.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 5866, "words": 9, "h1": "", "title": "Just a moment...", "challenge": true, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.perplexity.ai/robots.txt.", "status": 200, "finalUrl": "https://www.perplexity.ai/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 448, "sitemapUrls": [ "https://www.perplexity.ai/sitemap.xml", "https://www.perplexity.ai/help-center/sitemap.xml" ], "textLength": 448, "text": "# https://www.robotstxt.org/robotstxt.html\nUser-agent: *\nDisallow: /*?*q=\nDisallow: /search/new\nDisallow: /search?*/\nDisallow: /marketing/prerelease/\nDisallow: /onboarding/\nDisallow: /join/\nUser-agent: Googlebot\nDisallow: /*?*q=\nDisallow: /search*\nDisallow: /search?*/\nDisallow: /marketing/prerelease/\nDisallow: /onboarding/\nDisallow: /join/\nSitemap: https://www.perplexity.ai/sitemap.xml\nSitemap: https://www.perplexity.ai/help-center/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://www.perplexity.ai/llms.txt", "contentType": "text/html; charset=UTF-8", "bytes": 5890, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.perplexity.ai/sitemap.xml", "contentType": "application/xml; charset=utf-8", "bytes": 661, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "xai", "name": "xAI", "url": "https://x.ai", "category": "Assistant", "notable": true, "checkedAt": "2026-10-14T01:25:09.953Z", "result": { "ok": true, "inputUrl": "https://x.ai/", "finalUrl": "https://x.ai/", "checkedPath": "/", "pageStatus": 403, "truncated": false, "elapsedMs": 1726, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://x.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://x.ai/ · HTTP 403 · 5839 bytes. JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 127 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 403, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 5839, "words": 127, "h1": "Sorry, you have been blocked", "title": "Attention Required! | Cloudflare", "challenge": true, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://x.ai/robots.txt.", "status": 200, "finalUrl": "https://x.ai/robots.txt", "contentType": "text/plain; charset=UTF-8", "bytes": 225, "sitemapUrls": [ "https://x.ai/sitemap.xml" ], "textLength": 223, "text": "User-agent: *\nAllow: /\nDisallow: /tools/\n\nSitemap: https://x.ai/sitemap.xml\n\n# Content Signals — declare AI/search usage preferences\n# See: https://contentsignals.org/\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://x.ai/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 79738, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://x.ai/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 5839, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "copilot", "name": "Microsoft Copilot", "url": "https://copilot.microsoft.com", "category": "Assistant", "notable": true, "checkedAt": "2026-10-14T01:25:12.683Z", "result": { "ok": true, "inputUrl": "https://copilot.microsoft.com/", "finalUrl": "https://copilot.com/?fromcode=cmmlf487uwy&sessionId=05159787-c0a1-4068-b9a1-0002860c3ed7&", "checkedPath": "/?fromcode=cmmlf487uwy&sessionId=05159787-c0a1-4068-b9a1-0002860c3ed7&", "pageStatus": 200, "truncated": false, "elapsedMs": 2246, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "/robots.txt returned HTML, not a robots file.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. GPTBot is allowed by default.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. OAI-SearchBot is allowed by default.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. ChatGPT-User is allowed by default.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. ClaudeBot is allowed by default.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. PerplexityBot is allowed by default.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. Google-Extended is allowed by default.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. Applebot-Extended is allowed by default.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. CCBot is allowed by default.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "/robots.txt returned HTML, not a robots file. Bytespider is allowed by default.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. /robots.txt returned HTML, not a robots file.", "fix": "There is no official token to allow. Keep your positioning and pricing in the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://copilot.com/?fromcode=cmmlf487uwy&sessionId=05159787-c0a1-4068-b9a1-0002860c3ed7& · HTTP 200 · 304503 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1051 words in the raw HTML, ignoring script and style. H1: “Sign into Copilot”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: WebPage, Organization, ImageObject.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned an HTML page, not a text file.", "fix": "Serve a real plain-text file at /llms.txt. An HTML fallback does not count." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTML, not an XML urlset or sitemap index.", "fix": "Serve XML at /sitemap.xml (a urlset or sitemap index), not an HTML page." } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 304503, "words": 1051, "h1": "Sign into Copilot", "title": "Microsoft Copilot | Sign in", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "WebPage", "Organization", "ImageObject" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "default", "note": "/robots.txt returned HTML, not a robots file.", "status": 200, "finalUrl": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=4765445b-32c6-49b0-83e6-1d93765276ca&redirect_uri=https%3A%2F%2Fcopilot.com%2Flandingv2&response_type=code%20id_token&scope=openid%20profile%20offline_access%20https%3A%2F%2Fwww.office.com%2Fv2%2FOfficeHome.All&response_mode=form_post&nonce=639275379136124393.MzdjNWJhYmEtNDQwNS00ODQzLTliOWQtZjg4OTQ5M2E2NmNiNGY1MWIyZjMtOTdiMy00ZDU4LWIwZDYtNTUyN2FmZDRiYWYx&client-request-id=31e89ec2-add7-4d77-8dad-03fbe93bfc25&state=yx4LvJDXrB6aidfJ-5BW606qxWzAx4R1OwgA2c-VFXHE2uQWT1FEhnni4r3docF1Dk_X6JereJs4vA5p7Y0YGRLpPImQK_jFk0nGAVXdJyMAkVdaP6QFpOvB_3eSPKA-UQICodzbkkB7rTkqe6pt85tv2rUUs1yy48gt_vugAO83ZpQ3XXiC_Gv2Fn5HI6eawyHtIMKHL--jXvJrFCmo7jAjqC4GX5n8bmobiz7ifKzBqpT3P_m-PQYOBHv5LPkcgqWSZMT-xX-ylTI46m37P-lDhMGogEmXKGb9lEgB5M1cn-2bqotp4OWIf5eXbNc2FpZTjTjaFcT9EgruuJ84ZNLhHWb_E-c_bfmn5V1TM40bULBJsRUDXlm90yAyCEL-&x-client-SKU=ID_NET8_0&x-client-ver=8.14.0.0", "contentType": "text/html; charset=utf-8", "bytes": 53475, "sitemapUrls": [], "textLength": 53467, "text": "\r\n\r\n\r\n\r\n\r\n\r\n Sign in to your account\r\n \r\n \r\n \r\n \r\n \r\n \r\n\r\n\r\n\r\n\r\n \r\n \r\n \r\n \r\n\r\n \r\n\r\n \r\n\r\n \r\n\r\n \r\n \r\n\r\n\r\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" } }, "xai": { "GrokBot": { "allowed": null, "specific": false, "agent": "", "rule": "" }, "xAI-SearchBot": { "allowed": null, "specific": false, "agent": "", "rule": "" } }, "llms": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://www.midjourney.com/llms.txt", "contentType": "text/html; charset=UTF-8", "bytes": 5891, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://www.midjourney.com/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 5900, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "stability", "name": "Stability AI", "url": "https://stability.ai", "category": "Image", "notable": true, "checkedAt": "2026-10-14T01:25:36.790Z", "result": { "ok": true, "inputUrl": "https://stability.ai/", "finalUrl": "https://stability.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1701, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://stability.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: GPTBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: ClaudeBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: Google-Extended. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: Applebot-Extended. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: CCBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: Bytespider. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://stability.ai/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "warn", "detail": "919 words in the raw HTML, ignoring script and style. No H1 heading. Counted from the first 512 KB.", "fix": "Add a text H1 in the HTML so crawlers have a main heading. The word count itself is fine." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $232, $76.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "warn", "detail": "JSON-LD types: WebSite.", "fix": "Add SoftwareApplication or Product JSON-LD with an Offer so the name, description and price are explicit." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://stability.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html;charset=utf-8", "bytes": 524288, "words": 919, "h1": "", "title": "Stability AI", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [ "$232", "$76" ], "metaPrices": [], "ldPrices": [], "ldTypes": [ "WebSite" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://stability.ai/robots.txt.", "status": 200, "finalUrl": "https://stability.ai/robots.txt", "contentType": "text/plain;charset=utf-8", "bytes": 1503, "sitemapUrls": [ "https://stability.ai/sitemap.xml" ], "textLength": 1503, "text": "# Squarespace Robots Txt\n\nUser-agent: AI2Bot\nUser-agent: Ai2Bot-Dolma\nUser-agent: aiHitBot\nUser-agent: Amazonbot\nUser-agent: anthropic-ai\nUser-agent: Applebot-Extended\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ClaudeBot\nUser-agent: cohere-ai\nUser-agent: cohere-training-data-crawler\nUser-agent: DuckAssistBot\nUser-agent: FacebookBot\nUser-agent: Google-Extended\nUser-agent: GoogleOther\nUser-agent: GoogleOther-Image\nUser-agent: GoogleOther-Video\nUser-agent: GPTBot\nUser-agent: img2dataset\nUser-agent: Meta-ExternalAgent\nUser-agent: MyCentralAIScraperBot\nUser-agent: omgili\nUser-agent: omgilibot\nUser-agent: Quora-Bot\nUser-agent: TikTokSpider\nUser-agent: YouBot\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nUser-agent: AdsBot-Google-Mobile-Apps\nUser-agent: *\nDisallow: /config\nDisallow: /search\nDisallow: /account$\nDisallow: /account/\nDisallow: /commerce/digital-download/\nDisallow: /api/\nAllow: /api/ui-extensions/\nDisallow: /static/\nDisallow:/*?author=*\nDisallow:/*&author=*\nDisallow:/*?tag=*\nDisallow:/*&tag=*\nDisallow:/*?month=*\nDisallow:/*&month=*\nDisallow:/*?view=*\nDisallow:/*&view=*\nDisallow:/*?format=json\nDisallow:/*&format=json\nDisallow:/*?format=page-context\nDisallow:/*&format=page-context\nDisallow:/*?format=main-content\nDisallow:/*&format=main-content\nDisallow:/*?format=json-pretty\nDisallow:/*&format=json-pretty\nDisallow:/*?format=ical\nDisallow:/*&format=ical\nDisallow:/*?reversePaginate=*\nDisallow:/*&reversePaginate=*\n\nSitemap: https://stability.ai/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "GPTBot", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ClaudeBot", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Google-Extended", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Applebot-Extended", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "CCBot", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Bytespider", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://stability.ai/llms.txt", "contentType": "text/plain;charset=utf-8", "bytes": 0, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://stability.ai/sitemap.xml", "contentType": "text/xml;charset=utf-8", "bytes": 283772, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "runway", "name": "Runway", "url": "https://runwayml.com", "category": "Video", "notable": true, "checkedAt": "2026-10-14T01:25:39.496Z", "result": { "ok": true, "inputUrl": "https://runwayml.com/", "finalUrl": "https://runway.com/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1754, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://runway.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://runway.com/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "warn", "detail": "100 words in the raw HTML, ignoring script and style. H1: “Building Real-World Intelligence”. Counted from the first 512 KB.", "fix": "Add a plain-text H1 and a short paragraph (what it is, and who it's for) to the initial HTML." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, Person, ContactPoint, SoftwareApplication, WebSite.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 8035 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://runway.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 524288, "words": 100, "h1": "Building Real-World Intelligence", "title": "Runway | Building Real-World Intelligence", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "Organization", "Person", "ContactPoint", "SoftwareApplication", "WebSite" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://runway.com/robots.txt.", "status": 200, "finalUrl": "https://runway.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 666, "sitemapUrls": [ "https://runway.com/sitemap.xml" ], "textLength": 664, "text": "User-agent: *\nAllow: /\n\n# Gated assets. The landing pages at /reports/ are indexable; the PDFs\n# they give away are not. Each download response also carries an\n# `X-Robots-Tag: noindex` header, which is the enforcing half — this rule just\n# keeps well-behaved crawlers from spending requests on an endpoint that\n# answers 404 without a form submission.\nDisallow: /api/gated-resource/\n\n# Hosted files (/files/, Sanity `hostedFile`) are deliberately NOT listed\n# here: each is noindex or indexable per document, carried as an X-Robots-Tag on\n# its own response, and a blanket Disallow would override that opt-in.\n\nSitemap: https://runway.com/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://runway.com/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 8035, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://runway.com/sitemap.xml", "contentType": "application/xml", "bytes": 68659, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "leonardo", "name": "Leonardo.Ai", "url": "https://leonardo.ai", "category": "Image", "notable": false, "checkedAt": "2026-10-14T01:25:42.252Z", "result": { "ok": true, "inputUrl": "https://leonardo.ai/", "finalUrl": "https://leonardo.ai/", "checkedPath": "/", "pageStatus": 403, "truncated": false, "elapsedMs": 1962, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://leonardo.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://leonardo.ai/ · HTTP 403 · 5817 bytes. JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 9 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 403, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 5817, "words": 9, "h1": "", "title": "Just a moment...", "challenge": true, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://leonardo.ai/robots.txt.", "status": 200, "finalUrl": "https://leonardo.ai/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 270, "sitemapUrls": [ "https://leonardo.ai/sitemaps.xml" ], "textLength": 270, "text": "User-agent: *\nDisallow: /wp-admin/\nDisallow: /search/*\nDisallow: /?s=*\nDisallow: /*?s=\nDisallow: /*?__hstc=\nDisallow: /*?__hssc=\nDisallow: /*?__hsfp=\nDisallow: /*?hubs_content=\nDisallow: /*?utm_\nAllow: /wp-admin/admin-ajax.php\n\nSitemap: https://leonardo.ai/sitemaps.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://leonardo.ai/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 125977, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://leonardo.ai/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 5850, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "ideogram", "name": "Ideogram", "url": "https://ideogram.ai", "category": "Image", "notable": false, "checkedAt": "2026-10-14T01:25:45.218Z", "result": { "ok": true, "inputUrl": "https://ideogram.ai/", "finalUrl": "https://ideogram.ai/", "checkedPath": "/", "pageStatus": 403, "truncated": false, "elapsedMs": 1701, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://ideogram.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://ideogram.ai/ · HTTP 403 · 5817 bytes. JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 9 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 849 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 403, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 5817, "words": 9, "h1": "", "title": "Just a moment...", "challenge": true, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://ideogram.ai/robots.txt.", "status": 200, "finalUrl": "https://ideogram.ai/robots.txt", "contentType": "text/plain", "bytes": 1111, "sitemapUrls": [ "https://ideogram.ai/sitemap-index.xml" ], "textLength": 1101, "text": "# Allow social media bots to unfurl our share links on /g/ permalinks.\n# Each User-agent block is independent per the robots.txt spec, so the\n# Disallow: /g/ in the * block below does not affect these named bots —\n# they need an explicit Allow here to crawl /g/ for unfurl metadata.\nUser-agent: Discordbot\nAllow: /g/\n\nUser-agent: facebookexternalhit\nAllow: /g/\n\nUser-agent: LinkedInBot\nAllow: /g/\n\nUser-agent: Twitterbot\nAllow: /g/\n\nUser-agent: Googlebot\nAllow: /g/\n\n# Default crawler policy.\n# Allow the marketing surface (everything served by ideogram-ai/website).\n# Disallow app surfaces owned by ideogram-ai/ui:\n# /api/ — REST endpoints. Do not Allow /api/learn or /api/pricing; those\n# marketing dups were retired. Top-level /api-* 301s in nginx.\n# /g/ — generation permalinks (too many, low search value; social-bot\n# allow rules above carve out the unfurling case)\n# /t/ — gallery / feed / explore\n# /u/ — user profile pages\nUser-agent: *\nAllow: /\nDisallow: /api/\nDisallow: /g/\nDisallow: /t/\nDisallow: /u/\n\nSitemap: https://ideogram.ai/sitemap-index.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://ideogram.ai/llms.txt", "contentType": "text/plain", "bytes": 849, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://ideogram.ai/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 5871, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "firefly", "name": "Adobe Firefly", "url": "https://www.adobe.com/products/firefly.html", "category": "Image", "notable": true, "checkedAt": "2026-10-14T01:25:47.924Z", "result": { "ok": true, "inputUrl": "https://www.adobe.com/products/firefly.html", "finalUrl": "https://www.adobe.com/products/firefly.html", "checkedPath": "/products/firefly.html", "pageStatus": 200, "truncated": false, "elapsedMs": 3362, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /products/firefly.html. Fetched https://www.adobe.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /products/firefly.html. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.adobe.com/products/firefly.html · HTTP 200 · 63197 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1852 words in the raw HTML, ignoring script and style. H1: “Adobe Firefly. Create stunning content faster.”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 10499 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 404, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 63197, "words": 1852, "h1": "Adobe Firefly. Create stunning content faster.", "title": "Adobe Firefly - Free Generative AI for Creatives", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.adobe.com/robots.txt.", "status": 200, "finalUrl": "https://www.adobe.com/robots.txt", "contentType": "text/plain; charset=UTF-8", "bytes": 10199, "sitemapUrls": [ "https://www.adobe.com/cc-product.index.xml", "https://www.adobe.com/creativecloud/sitemap-index.xml", "https://www.adobe.com/acrobat/sitemap-index.xml", "https://www.adobe.com/home-sitemap.xml", "https://www.adobe.com/plans-catalog-sitemap.xml", "https://www.adobe.com/learn/sitemap_index.xml", "https://www.adobe.com/express/sitemap-index.xml", "https://www.adobe.com/naver.index.xml", "https://www.adobe.com/edu/sitemap-index.xml" ], "textLength": 10199, "text": "# The use of robots or other automated means to access the Adobe site\n# without the express permission of Adobe is strictly prohibited.\n# Notwithstanding the foregoing, Adobe may permit automated access to\n# access certain Adobe pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at http://www.robotstxt.org/ is strictly prohibited.\n\n# Details about Googlebot available at: http://www.google.com/bot.html\n# The Google search engine can see everything\nUser-agent: gsa-crawler-www \nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*/globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dotcom/*.html$\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020/digital-trends-2020-full-report.pdf\nDisallow: /plans-fragments/\nDisallow: /plans.modals*\n\n# The Omniture search engine can see everything\nUser-agent: Atomz/1.0\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /products/flashplayer/distribution3.html\nDisallow: /products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\n\nUser-agent: *\nAllow: /assets/globalnav/v1.1.4/css/globalnav.css\nAllow: /assets/globalnav/v1.1.4/*globalnav.js\nAllow: /au/special/symposium\nAllow: /sea/special/symposium\nAllow: /*special/products/marketing-cloud/*.css$\nAllow: /*special/products/marketing-cloud/*.js$\nAllow: /*special/products/marketing-cloud/*.png$\nAllow: /www.adobe.com/special/ri/prod/assets/*.css$\nAllow: /www.adobe.com/special/ri/prod/assets/*.jpg$\nAllow: /www.adobe.com/special/ri/prod/assets/*.js$\nAllow: /www.adobe.com/special/ri/prod/assets/*.png$\nAllow: /www.adobe.com/special/retail-shopping-insights/\nAllow: /uk/special/landing/student/digitaledgeaward/\nAllow: /uk/special/landing/student/lovethejourney\nAllow: /fr/special/landing/student/lovethejourney\nAllow: /community/gallery$\nAllow: /community/gallery/assets$\nAllow: /community/gallery/challenges$\nDisallow: /assets/globalnav/\nDisallow: /*creativesuite/bridge\nDisallow: /cfusion/gpr/\nDisallow: /cfusion/search/\nDisallow: /cfusion/partnerportal/\nDisallow: /globalsearch\nDisallow: /cgi-local/\nDisallow: /de/products/index-notabs.html\nDisallow: /de/products/index-tabs.html\nDisallow: /devnet-archive/\nDisallow: /devnet/archive\nDisallow: /*devnet/tagsearch\nDisallow: /emea/eemeachannel/\nDisallow: /events/ciocouncil/\nDisallow: /events/executivecouncil/\nDisallow: /google_indexing/\nDisallow: /*.html_\nDisallow: /licensing/distribution/strategies/\nDisallow: /*limited/\nDisallow: /livedocs/\nDisallow: /*Ljava.lang.Object*\nDisallow: /products/acrobat/readstep2.html;JMICROSITE2SESSION*\nDisallow: /*products/flashplayer/distribution3.html\nDisallow: /*products/flashplayer/distribution4.html\nDisallow: /*special/\nDisallow: /*.sql$\nDisallow: /*supportservice/\nDisallow: /*.translate.html\nDisallow: /*products/adobeconnect/enterprise/trial.html\nDisallow: /*products/adobeconnect/enterprise/trial-check.html\nDisallow: /*products/adobeconnect/enterprise/trial-thankyou.html\nDisallow: /*products/adobeconnect/enterprise/trial-error.html\nDisallow: /*products/photoshop/selector\nDisallow: /*unsubscribe.html\nDisallow: /content/dam/acom/en/marketing-cloud/offers/pdf\nDisallow: /content/dotcom/*.html$\nDisallow: /plans-fragments/\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-DE.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN-US.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-EN.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-ES.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-FR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-IT.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-JA.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-KR.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hans.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital-trends-2020-in-media-and-entertainment-zh-Hant.pdf\nDisallow: /content/dam/www/us/en/offer/digital-trends-2020-in-media-and-entertainment-html/digital_trends_21_core_report_TC_FA.pdf\nDisallow: /content/dam/dx/us/en/resources/reports/digital-trends-2021-in-high-tech/2021_Digital_Trends_High_Tech_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-telecommunications-media-and-entertainment/Digital_Trends_2021_Media_Entertainment_and_Telecoms_Report.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-retail/2021_Digital_Trends_Retail_in_Focus.pdf\nDisallow: /content/dam/dx/us/en/resources/digital-trends-2021-in-manufacturing/2021_Digital_Trends_Manufacturing_in_Focus.pdf\nDisallow: /content/dam/www/us/en/offer/technology-trends-2021/eConsultancy_Digital_Trends_2021_IT_Report.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-financial-services.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-retail.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-media-and-entertainment.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-travel-and-hospitality.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-telecommunications.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-advertising.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-high-tech.pdf\nDisallow: /content/dam/www/us/en/offer/pdf/digital-trends-2020-in-IT.pdf\nDisallow: /content/dam/www/us/en/adi/2020/pdfs/*\nDisallow: /content/dam/www/us/en/offer/digital-trends-2021-core/digital-trends-2021-full-report-*.pdf\nDisallow: /*langmaster/\nDisallow: /*express/*.mp4$\nDisallow: /en-AE/learn/\nDisallow: /en-BE/learn/\nDisallow: /en-CY/learn/\nDisallow: /en-GR/learn/\nDisallow: /en-HK/learn/\nDisallow: /en-ID/learn/\nDisallow: /en-IL/learn/\nDisallow: /en-LU/learn/\nDisallow: /en-MT/learn/\nDisallow: /en-MY/learn/\nDisallow: /en-PH/learn/\nDisallow: /en-SA/learn/\nDisallow: /en-SG/learn/\nDisallow: /en-TH/learn/\nDisallow: /en-VN/learn/\nDisallow: /en-x-MenA/learn/\nDisallow: /en-ZA/learn/\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\nDisallow: /community/gallery\n\nUser-agent: Googlebot\nAllow: /?promoid=RTQCN3LX\nDisallow: /*promoid=\nDisallow: /*trackingid=\nDisallow: /*?searchId=\nDisallow: /*?tasks=\nDisallow: /*?campaignid=\nDisallow: /*?skwcid=\nDisallow: /products/photoshop/photoshop-brushes.h*\n\nUser-agent: AdsBot-Google\nAllow: /\n\nUser-agent: AdsBot-Google-Mobile\nAllow: /\n\nUser-agent: SearchmetricsBot\nDisallow: /\n\n# XML sitemaps\nSitemap: https://www.adobe.com/cc-product.index.xml\nSitemap: https://www.adobe.com/creativecloud/sitemap-index.xml\nSitemap: https://www.adobe.com/acrobat/sitemap-index.xml\nSitemap: https://www.adobe.com/home-sitemap.xml\nSitemap: https://www.adobe.com/plans-catalog-sitemap.xml\nSitemap: https://www.adobe.com/learn/sitemap_index.xml\n\n# CC Express \nSitemap: https://www.adobe.com/express/sitemap-index.xml\n\n#Naver\nSitemap: https://www.adobe.com/naver.index.xml\n\n#Education\nSitemap: https://www.adobe.com/edu/sitemap-index.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.adobe.com/llms.txt", "contentType": "text/plain; charset=UTF-8", "bytes": 10499, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.adobe.com/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 889, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "canva", "name": "Canva", "url": "https://www.canva.com", "category": "Design", "notable": true, "checkedAt": "2026-10-14T01:25:52.291Z", "result": { "ok": true, "inputUrl": "https://www.canva.com/", "finalUrl": "https://www.canva.com/", "checkedPath": "/", "pageStatus": 403, "truncated": true, "elapsedMs": 1750, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.canva.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "warn", "detail": "Blocked for / by User-agent: GPTBot (Disallow: /).", "fix": "Leave the block if you don't want training use. To be cited in AI answers, allow the search crawlers (OAI-SearchBot, PerplexityBot)." }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "warn", "detail": "Blocked for / by User-agent: ClaudeBot (Disallow: /).", "fix": "Leave the block if you don't want training use. To be cited in AI answers, allow the search crawlers (OAI-SearchBot, PerplexityBot)." }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: PerplexityBot.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "warn", "detail": "Blocked for / by User-agent: Applebot-Extended (Disallow: /).", "fix": "Leave this Disallow if you want to opt out of Apple Intelligence training. It does not block ordinary search crawling." }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "warn", "detail": "Blocked for / by User-agent: CCBot (Disallow: /).", "fix": "Leave the block if you don't want training use. To be cited in AI answers, allow the search crawlers (OAI-SearchBot, PerplexityBot)." }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "warn", "detail": "Blocked for / by User-agent: Bytespider (Disallow: /).", "fix": "Leave the block if you don't want training use. To be cited in AI answers, allow the search crawlers (OAI-SearchBot, PerplexityBot)." }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.canva.com/ · HTTP 403 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 32 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 403.", "fix": "Serve /llms.txt as a public plain-text file with HTTP 200." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 403, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 524288, "words": 32, "h1": "We’ll have you designing again soon", "title": "Canva", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.canva.com/robots.txt.", "status": 200, "finalUrl": "https://www.canva.com/robots.txt", "contentType": "text/plain;charset=UTF-8", "bytes": 6527, "sitemapUrls": [ "https://sitemap.canva.com/sitemap_index.xml", "https://www.canva.com/landing_page_sitemap_index.xml" ], "textLength": 6527, "text": "# vnacccan\n# nVCCCNVccNCCNv\n# cCCCAa aCCC\n# nCCCCa nCCC\n# cCCCAv ACCa CACCn\n# aCCCCv nVa vcVNn nVVa cVc aVNVv cNNv Cc Cc vaVNVa vcVa\n# CCCCa nACCAcaAACCCc nCCCa aCCCCCC NCCCv Cc Cn cNCCCVaNACCCA\n# aCCCC cCCCV cCCCA VCCCaAA CCCA CCCCv CACNvnVCCCAv vCCCCv\n# cCCCC aCCCN VCCCa vCCCCCV CCCc CCCCv aCA cCCCCv nCCCN\n# aCCCC nCCCCv CCCA ACCCCc CCC nCCCCn vCN VCCCc NCCCn\n# aCCCCa aCCCCC NCCCc NCCCCV CCCN C aCCCc aCc NCCCn aCCCA Nc\n# aCCCCn nNV CCCCa nAACCCc AACCCN vCCCAaVC ACCCaNAv cCCCN vVACCCA vNA\n# aCCCCNav vncACc NCCCNn VCCCcVvvCCCv cCCCCV vACCNn cCCCCAa aCCCAa\n# aNCCCCCCCCNa\n#\n# See something you can improve? We're hiring SEOs and Engineers!\n# Checkout canva.com/careers and get in touch\nUser-agent: *\nDisallow: \nDisallow: /media/*\nDisallow: /template/*\nDisallow: /_ok\nDisallow: /_blank\nDisallow: *v=\nDisallow: *utm_expid=\nDisallow: *source=\nDisallow: *utm_source=\nDisallow: *utm_campaign=\nDisallow: *utm_content=\nDisallow: *__hstc=\nDisallow: *reviews_page=\nDisallow: *gclid=\nDisallow: *magazineName=\nDisallow: *_ga=\nDisallow: *like=\nDisallow: *sp_url=\nDisallow: *fbclid=\nDisallow: *country=\nDisallow: *company_size=\nDisallow: *_hsenc=\nDisallow: *zd_source=\nDisallow: *wt.mc_id=\nDisallow: *via=\nDisallow: *utm_medium=\nDisallow: *spc-source=\nDisallow: *sp=\nDisallow: *sort=\nDisallow: *sf11500465=\nDisallow: *sa=\nDisallow: *ref=\nDisallow: *preview=\nDisallow: *pr=\nDisallow: *o=\nDisallow: *NoCode=\nDisallow: *mod=\nDisallow: *kui=\nDisallow: *industry=\nDisallow: *iframe=\nDisallow: *hsLang=\nDisallow: *hsCtaTracking=\nDisallow: *hash=\nDisallow: *gh_jid=\nDisallow: *filterTags=\nDisallow: *clickId=\nDisallow: *ca_referer=\nDisallow: *author=\nDisallow: *ak_action=\nDisallow: *ad=\nDisallow: *__hstc=\nDisallow: *__+hsfp=\nDisallow: /_ajax/\nDisallow: /design/\nDisallow: /design?create*\nDisallow: /font-combinations/search/*\nDisallow: /*/followers\nDisallow: /*/followers/\nDisallow: /*/following\nDisallow: /*/following/\nDisallow: /templates/*/*/?uid=*\nDisallow: /templates/?uid=*\nAllow: /templates/classroom-decoration\nAllow: /t/E\nDisallow: /templates/M\nDisallow: /pt_br/modelos/M\nDisallow: /es_es/plantillas/M\nDisallow: /es_mx/plantillas/M\nDisallow: /fr_fr/modeles/M\nDisallow: /de_de/vorlagen/M\nDisallow: /ru_ru/shablony/M\nDisallow: /tr_tr/sablonlar/M\nDisallow: /ja_jp/templates/M\nDisallow: /ar_eg/templates/M\nDisallow: /id_id/contoh/M\nDisallow: /es_ar/plantillas/M\nDisallow: /it_it/modelli/M\nDisallow: /es_co/plantillas/M\nDisallow: /pl_pl/szablony/M\nDisallow: /th_th/templates/M\nDisallow: /pt_pt/modelos/M\nDisallow: /vi_vn/mau/M\nDisallow: /nl_nl/sjablonen/M\nDisallow: /ko_kr/templates/M\nDisallow: /sv_se/mallar/M\nDisallow: /zh_tw/templates/M\nDisallow: /hi_in/templates/M\nDisallow: /templates/*/MA\nDisallow: /*/_m2/\nDisallow: /_upload-widget\nDisallow: /*/_upload-widget\nDisallow: /_print/*\nDisallow: /en_oz/\nDisallow: /join/*\nDisallow: /q/*\nDisallow: *?filters=\nDisallow: *&filters=\nDisallow: /*/help/*?query=\nDisallow: /*/help/*&query=\nDisallow: /zh_cn/\nDisallow: /en_psaccent/\nDisallow: /en_instrume/\nDisallow: /ar/\nDisallow: /ar_ae/\nDisallow: /ar_sa/\nDisallow: /en_ph/\nDisallow: /cdn-cgi/\nDisallow: /*/login/verify-email\nDisallow: /M/\n\nUser-agent: AdsBot-Google\nUser-agent: AdsIdxBot\nUser-agent: Google-InspectionTool\nUser-agent: facebookexternalhit\nUser-agent: LinkedInBot\nUser-agent: PerplexityBot\nUser-agent: Perplexity-User\nUser-agent: Canva-Slackapp-LinkExpanding\nUser-agent: Canva-Teamsapp-LinkExpanding\nAllow: /\n\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: CCBot\nUser-agent: Applebot-Extended\nUser-agent: anthropic-ai\nUser-agent: Claude-Web\nDisallow: /\nAllow: /create/\nAllow: /ar_eg/create/\nAllow: /da_dk/skabe/\nAllow: /de_de/erstellen/\nAllow: /es_ar/crear/\nAllow: /es_co/crear/\nAllow: /es_es/crear/\nAllow: /es_mx/crear/\nAllow: /es_us/crear/\nAllow: /fr_fr/creer/\nAllow: /hi_in/banaye/\nAllow: /id_id/membuat/\nAllow: /it_it/creare/\nAllow: /ja_jp/create/\nAllow: /ko_kr/create/\nAllow: /ms_my/buat/\nAllow: /nb_no/skape/\nAllow: /nl_nl/maken/\nAllow: /pl_pl/tworzyc/\nAllow: /pt_br/criar/\nAllow: /pt_pt/criar/\nAllow: /ru_ru/sozdat/\nAllow: /sv_se/skapa/\nAllow: /th_th/create/\nAllow: /tr_tr/yapma/\nAllow: /uk_ua/stvoryty/\nAllow: /vi_vn/thiet-ke/\nAllow: /zh_hk/create/\nAllow: /zh_tw/create/\nAllow: /features/\nAllow: /ar_eg/features/\nAllow: /da_dk/funktioner/\nAllow: /de_de/funktionen/\nAllow: /es_ar/funciones/\nAllow: /es_co/funciones/\nAllow: /es_es/funciones/\nAllow: /es_mx/funciones/\nAllow: /es_us/funciones/\nAllow: /fr_fr/outils/\nAllow: /hi_in/features/\nAllow: /id_id/fitur/\nAllow: /it_it/strumenti/\nAllow: /ja_jp/features/\nAllow: /ko_kr/features/\nAllow: /nl_nl/functies/\nAllow: /pt_pt/ferramentas/\nAllow: /sv_se/funktioner/\nAllow: /th_th/features/\nAllow: /tr_tr/ozellikler/\nAllow: /vi_vn/tinh-nang/\nAllow: /zh_hk/features/\nAllow: /zh_tw/features/\nAllow: /graphs/\nAllow: /ar_eg/graphs/\nAllow: /da_dk/grafer/\nAllow: /de_de/diagramme/\nAllow: /es_ar/graficos/\nAllow: /es_co/graficos/\nAllow: /es_es/graficos/\nAllow: /es_mx/graficas/\nAllow: /es_us/graficas/\nAllow: /fr_fr/graphiques/\nAllow: /hi_in/graphs/\nAllow: /it_it/diagramma/\nAllow: /ja_jp/graphs/\nAllow: /ko_kr/graphs/\nAllow: /nb_no/grafer/\nAllow: /nl_nl/grafieken/\nAllow: /pt_br/graficos/\nAllow: /pt_pt/graficos/\nAllow: /sv_se/grafer/\nAllow: /th_th/graphs/\nAllow: /tr_tr/grafik/\nAllow: /tr_tr/sema/\nAllow: /vi_vn/bieu-do/\nAllow: /zh_hk/graphs/\nAllow: /zh_tw/graphs/\nAllow: /learn/\nAllow: /de_de/lernen/\nAllow: /fr_fr/decouvrir/\nAllow: /ja_jp/learn/\nAllow: /ko_kr/learn/\nAllow: /pt_br/aprenda/\nAllow: /ru_ru/obuchenie/\nAllow: /help/\nAllow: /*/help/\nAllow: /design-school/\nAllow: /*/design-school/\nAllow: /ai-connector/\nAllow: /*/ai-connector/\nAllow: /developers/\n\nUser-agent: Bytespider\nUser-agent: meta-externalagent\nUser-agent: ImagesiftBot\nUser-agent: Timpibot\nUser-agent: Scrapy\nUser-agent: Ai2Bot\nUser-agent: Diffbot\nUser-agent: img2dataset\nUser-agent: ICC-Crawler\nUser-agent: FacebookBot\nUser-agent: VelenPublicWebCrawler\nUser-agent: FriendlyCrawler\nUser-agent: omgili\nUser-agent: Webzio-Extended\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Twitterbot\nAllow: /\n\n\nSitemap: https://sitemap.canva.com/sitemap_index.xml\nSitemap: https://www.canva.com/landing_page_sitemap_index.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": false, "countedAsBlock": true, "specific": true, "agent": "GPTBot", "rule": "Disallow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": false, "countedAsBlock": true, "specific": true, "agent": "ClaudeBot", "rule": "Disallow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "PerplexityBot", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": false, "countedAsBlock": true, "specific": true, "agent": "Applebot-Extended", "rule": "Disallow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": false, "countedAsBlock": true, "specific": true, "agent": "CCBot", "rule": "Disallow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": false, "countedAsBlock": true, "specific": true, "agent": "Bytespider", "rule": "Disallow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://www.canva.com/llms.txt", "contentType": "text/html; charset=UTF-8", "bytes": 512000, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://www.canva.com/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 512000, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "photoroom", "name": "Photoroom", "url": "https://www.photoroom.com", "category": "Image", "notable": false, "checkedAt": "2026-10-14T01:25:55.046Z", "result": { "ok": true, "inputUrl": "https://www.photoroom.com/", "finalUrl": "https://www.photoroom.com/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1822, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.photoroom.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.photoroom.com/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1197 words in the raw HTML, ignoring script and style. H1: “Sell at first sight”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 5558 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.photoroom.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html", "bytes": 524288, "words": 1197, "h1": "Sell at first sight", "title": "AI-Powered Photo Editor and Listing Studio for Product Photography | Photoroom", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.photoroom.com/robots.txt.", "status": 200, "finalUrl": "https://www.photoroom.com/robots.txt", "contentType": "text/plain", "bytes": 87, "sitemapUrls": [ "https://www.photoroom.com/sitemap.xml" ], "textLength": 87, "text": "User-agent: *\nSitemap: https://www.photoroom.com/sitemap.xml\nAllow: /\nDisallow: /join/\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.photoroom.com/llms.txt", "contentType": "text/plain", "bytes": 5558, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.photoroom.com/sitemap.xml", "contentType": "text/xml", "bytes": 45327, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "krea", "name": "Krea", "url": "https://www.krea.ai", "category": "Image", "notable": false, "checkedAt": "2026-10-14T01:25:57.874Z", "result": { "ok": true, "inputUrl": "https://www.krea.ai/", "finalUrl": "https://www.krea.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1928, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.krea.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.krea.ai/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1515 words in the raw HTML, ignoring script and style. H1: “Krea.ai is the world's most powerful creative AI suite.”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $0, $9, $35, $105.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, ImageObject, WebSite, SoftwareApplication, Offer, BreadcrumbList, ListItem.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 23663 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.krea.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html", "bytes": 524288, "words": 1515, "h1": "Krea.ai is the world's most powerful creative AI suite.", "title": "Krea: AI Creative Suite for Images, Video & 3D", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [ "$0", "$9", "$35", "$105" ], "metaPrices": [], "ldPrices": [ "USD 0" ], "ldTypes": [ "Organization", "ImageObject", "WebSite", "SoftwareApplication", "Offer", "BreadcrumbList", "ListItem" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.krea.ai/robots.txt.", "status": 200, "finalUrl": "https://www.krea.ai/robots.txt", "contentType": "text/plain", "bytes": 278, "sitemapUrls": [ "https://www.krea.ai/sitemap.xml", "https://www.krea.ai/flush-sitemap.xml", "https://www.krea.ai/blog/sitemap.xml", "https://www.krea.ai/docs/sitemap.xml" ], "textLength": 278, "text": "# https://www.robotstxt.org/robotstxt.html\nUser-agent: *\nDisallow: /admin/\nDisallow: /*?preview=\n\nSitemap: https://www.krea.ai/sitemap.xml\nSitemap: https://www.krea.ai/flush-sitemap.xml\nSitemap: https://www.krea.ai/blog/sitemap.xml\nSitemap: https://www.krea.ai/docs/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.krea.ai/llms.txt", "contentType": "text/plain", "bytes": 23663, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.krea.ai/sitemap.xml", "contentType": "application/xml", "bytes": 427226, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "luma", "name": "Luma AI", "url": "https://lumalabs.ai", "category": "Video", "notable": false, "checkedAt": "2026-10-14T01:26:00.807Z", "result": { "ok": true, "inputUrl": "https://lumalabs.ai/", "finalUrl": "https://lumalabs.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 2093, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://lumalabs.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://lumalabs.ai/ · HTTP 200 · 304928 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "523 words in the raw HTML, ignoring script and style. H1: “Luma is your creative partner”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "A price is present in structured data (USD 0) but not in the visible text.", "fix": "Put the price in the visible HTML as well. Some crawlers never read JSON-LD." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: SoftwareApplication, Offer, Organization, VideoObject, ItemList, ListItem, PostalAddress, QuantitativeValue, ContactPoint, WebSite.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 2886 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://lumalabs.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 304928, "words": 523, "h1": "Luma is your creative partner", "title": "Luma | AI Agents for Creative Work", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [ "USD 0" ], "ldTypes": [ "SoftwareApplication", "Offer", "Organization", "VideoObject", "ItemList", "ListItem", "PostalAddress", "QuantitativeValue", "ContactPoint", "WebSite" ], "ldParsed": 5, "ldBlocks": 5, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://lumalabs.ai/robots.txt.", "status": 200, "finalUrl": "https://lumalabs.ai/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 761, "sitemapUrls": [ "https://lumalabs.ai/sitemap.xml", "https://lumalabs.ai/authors/sitemap.xml", "https://lumalabs.ai/case-studies/sitemap.xml", "https://lumalabs.ai/changelog/sitemap.xml", "https://lumalabs.ai/comparison/sitemap.xml", "https://lumalabs.ai/create/sitemap.xml", "https://lumalabs.ai/learning-center/sitemap.xml", "https://lumalabs.ai/legal/sitemap.xml", "https://lumalabs.ai/models/sitemap.xml", "https://lumalabs.ai/news/sitemap.xml", "https://lumalabs.ai/reframe/sitemap.xml", "https://lumalabs.ai/use-case/sitemap.xml", "https://lumalabs.ai/video-to-video/sitemap.xml" ], "textLength": 761, "text": "# *\nUser-agent: *\nAllow: /\nDisallow: /captures\nDisallow: /capture\nDisallow: /dashboard\nDisallow: /me\n\n# Sitemaps\nSitemap: https://lumalabs.ai/sitemap.xml\nSitemap: https://lumalabs.ai/authors/sitemap.xml\nSitemap: https://lumalabs.ai/case-studies/sitemap.xml\nSitemap: https://lumalabs.ai/changelog/sitemap.xml\nSitemap: https://lumalabs.ai/comparison/sitemap.xml\nSitemap: https://lumalabs.ai/create/sitemap.xml\nSitemap: https://lumalabs.ai/learning-center/sitemap.xml\nSitemap: https://lumalabs.ai/legal/sitemap.xml\nSitemap: https://lumalabs.ai/models/sitemap.xml\nSitemap: https://lumalabs.ai/news/sitemap.xml\nSitemap: https://lumalabs.ai/reframe/sitemap.xml\nSitemap: https://lumalabs.ai/use-case/sitemap.xml\nSitemap: https://lumalabs.ai/video-to-video/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://lumalabs.ai/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 2886, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://lumalabs.ai/sitemap.xml", "contentType": "application/xml", "bytes": 3883, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "pika", "name": "Pika", "url": "https://pika.art", "category": "Video", "notable": false, "checkedAt": "2026-10-14T01:26:03.903Z", "result": { "ok": true, "inputUrl": "https://pika.art/", "finalUrl": "https://pika.art/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1854, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://pika.art/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://pika.art/ · HTTP 200 · 342128 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "916 words in the raw HTML, ignoring script and style. H1: “AI Creative Tools Built for Creatives”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, WebSite.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 4213 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://pika.art/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 342128, "words": 916, "h1": "AI Creative Tools Built for Creatives", "title": "Pika", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "Organization", "WebSite" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://pika.art/robots.txt.", "status": 200, "finalUrl": "https://pika.art/robots.txt", "contentType": "text/plain", "bytes": 287, "sitemapUrls": [ "https://pika.art/sitemap.xml" ], "textLength": 287, "text": "User-Agent: *\nAllow: /\nDisallow: /api/\nDisallow: /auth/\nDisallow: /oauth/\nDisallow: /login\nDisallow: /signup\nDisallow: /onboarding\nDisallow: /account\nDisallow: /projects\nDisallow: /library\nDisallow: /credits\nDisallow: /jobs\nDisallow: /subscription\n\nSitemap: https://pika.art/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://pika.art/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 4213, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://pika.art/sitemap.xml", "contentType": "application/xml", "bytes": 2188, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "kling", "name": "Kling AI", "url": "https://klingai.com", "category": "Video", "notable": false, "checkedAt": "2026-10-14T01:26:06.761Z", "result": { "ok": true, "inputUrl": "https://klingai.com/", "finalUrl": "https://kling.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 3200, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://kling.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: GPTBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: OAI-SearchBot.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ChatGPT-User. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: (default). This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: PerplexityBot.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Google-Extended. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: (default). This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: (default). This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Bytespider. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://kling.ai/ · HTTP 200 · 102886 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "993 words in the raw HTML, ignoring script and style. H1: “Kling AI Video and Image Generator”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, ImageObject, WebSite, WebPage, SoftwareApplication, ItemList, ListItem, BreadcrumbList.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 22277 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://kling.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 102886, "words": 993, "h1": "Kling AI Video and Image Generator", "title": "Kling AI: Next-Gen AI Video & Image Generator", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "Organization", "ImageObject", "WebSite", "WebPage", "SoftwareApplication", "ItemList", "ListItem", "BreadcrumbList" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://kling.ai/robots.txt.", "status": 200, "finalUrl": "https://kling.ai/robots.txt", "contentType": "text/plain; charset=UTF-8", "bytes": 1549, "sitemapUrls": [ "https://kling.ai/sitemap.xml" ], "textLength": 1549, "text": "User-agent: GPTBot\nUser-agent: SemrushBot\nAllow: /\n\nUser-agent: Bytespider\nUser-agent: PanguBot\nUser-agent: PetalBot\nAllow: /\n\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nUser-agent: PerplexityBot\nUser-agent: Google-Extended\nUser-agent: bingbot\nUser-agent: YouBot\nUser-agent: Webzio-Extended\nUser-agent: Timpibot\nAllow: /\nDisallow: /app/sign-in\nDisallow: /app/search\nDisallow: /app/face-model\nDisallow: /app/text-to-image\nDisallow: /app/image-to-image\nDisallow: /app/text-to-video\nDisallow: /app/text-to-audio\nDisallow: /app/video-to-audio\nDisallow: /app/image-to-video\nDisallow: /app/multimodal-to-video\nDisallow: /app/special-effects\nDisallow: /app/video-to-lip\nDisallow: /app/membership\nDisallow: /app/user-assets\nDisallow: /app/user-profile\nDisallow: /app/account\nDisallow: /app/reward-tasks\nDisallow: /app/badge-center\nDisallow: /app/dev-center\nDisallow: /app/future-partner\nDisallow: /app/activity-zone/brand-activity\nDisallow: /app/skit-detail/\nDisallow: /app/work-detail/\nDisallow: /app/try-on\nDisallow: /app/docs/professional\n\nUser-agent: img2dataset\nUser-agent: Scrapy\nUser-agent: FriendlyCrawler\nUser-agent: ICC-Crawler\nUser-agent: ISSCyberRiskCrawler\nUser-agent: Sidetrade indexer bot\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: Googlebot\nUser-agent: Googlebot-Image\nUser-agent: Googlebot-Video\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nUser-agent: Mediapartners-Google\nUser-agent: Googlebot-News\nAllow: /\n\nSitemap: https://kling.ai/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "GPTBot", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "OAI-SearchBot", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ChatGPT-User", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "(default)", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "PerplexityBot", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Google-Extended", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "(default)", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "(default)", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Bytespider", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "(default)", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "(default)", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://kling.ai/llms.txt", "contentType": "text/plain", "bytes": 22277, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://kling.ai/sitemap.xml", "contentType": "text/xml", "bytes": 1096, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "elevenlabs", "name": "ElevenLabs", "url": "https://elevenlabs.io", "category": "Audio", "notable": true, "checkedAt": "2026-10-14T01:26:10.967Z", "result": { "ok": true, "inputUrl": "https://elevenlabs.io/", "finalUrl": "https://elevenlabs.io/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1809, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://elevenlabs.io/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://elevenlabs.io/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1532 words in the raw HTML, ignoring script and style. H1: “Bringing technology to life”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $1.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, ImageObject, PostalAddress, Person, WebSite, Brand, WebPage, BreadcrumbList, ListItem, SoftwareApplication, PropertyValue, Offer.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 11314 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://elevenlabs.io/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 524288, "words": 1532, "h1": "Bringing technology to life", "title": "Free AI Voice Generator & Voice Agents Platform | ElevenLabs", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [ "$1" ], "metaPrices": [], "ldPrices": [ "USD 0" ], "ldTypes": [ "Organization", "ImageObject", "PostalAddress", "Person", "WebSite", "Brand", "WebPage", "BreadcrumbList", "ListItem", "SoftwareApplication", "PropertyValue", "Offer" ], "ldParsed": 2, "ldBlocks": 2, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://elevenlabs.io/robots.txt.", "status": 200, "finalUrl": "https://elevenlabs.io/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 3372, "sitemapUrls": [ "https://elevenlabs.io/sitemap.xml", "https://help.elevenlabs.io/hc/sitemap.xml", "https://elevenlabs.io/docs/sitemap.xml", "https://showcase.elevenlabs.io/sitemap.xml" ], "textLength": 3372, "text": "User-Agent: Twitterbot\nAllow: /\nAllow: /share/\nAllow: /embed/\nAllow: /app/sign-in\nAllow: /app/sign-up\nAllow: /docs/api/fern-docs/og\nDisallow: /admin\nDisallow: /app/redirect-external/\nDisallow: /legal/\nDisallow: /hog/\nDisallow: /dachshund/\nDisallow: /feature-flags/\nDisallow: /app/sign-in?\nDisallow: /app/sign-up?\nDisallow: */abtest/\nDisallow: */p/\nDisallow: */ab/\nDisallow: /sentry-monitoring\nDisallow: /next/\nDisallow: */page-template-preview/\nDisallow: /v1/\nDisallow: /nl$\nDisallow: /nl/\nAllow: /app/flows\nDisallow: /app/agents\nDisallow: /app/api\nDisallow: /app/developers\nDisallow: /app/files\nDisallow: /app/settings\nDisallow: /app/subscription\n\nUser-Agent: facebookexternalhit\nAllow: /\nAllow: /share/\nAllow: /embed/\nAllow: /app/sign-in\nAllow: /app/sign-up\nAllow: /docs/api/fern-docs/og\nDisallow: /admin\nDisallow: /app/redirect-external/\nDisallow: /legal/\nDisallow: /hog/\nDisallow: /dachshund/\nDisallow: /feature-flags/\nDisallow: /app/sign-in?\nDisallow: /app/sign-up?\nDisallow: */abtest/\nDisallow: */p/\nDisallow: */ab/\nDisallow: /sentry-monitoring\nDisallow: /next/\nDisallow: */page-template-preview/\nDisallow: /v1/\nDisallow: /nl$\nDisallow: /nl/\nAllow: /app/flows\nDisallow: /app/agents\nDisallow: /app/api\nDisallow: /app/developers\nDisallow: /app/files\nDisallow: /app/settings\nDisallow: /app/subscription\n\nUser-Agent: LinkedInBot\nAllow: /\nAllow: /share/\nAllow: /embed/\nAllow: /app/sign-in\nAllow: /app/sign-up\nAllow: /docs/api/fern-docs/og\nDisallow: /admin\nDisallow: /app/redirect-external/\nDisallow: /legal/\nDisallow: /hog/\nDisallow: /dachshund/\nDisallow: /feature-flags/\nDisallow: /app/sign-in?\nDisallow: /app/sign-up?\nDisallow: */abtest/\nDisallow: */p/\nDisallow: */ab/\nDisallow: /sentry-monitoring\nDisallow: /next/\nDisallow: */page-template-preview/\nDisallow: /v1/\nDisallow: /nl$\nDisallow: /nl/\nAllow: /app/flows\nDisallow: /app/agents\nDisallow: /app/api\nDisallow: /app/developers\nDisallow: /app/files\nDisallow: /app/settings\nDisallow: /app/subscription\n\nUser-Agent: WhatsApp\nAllow: /\nAllow: /share/\nAllow: /embed/\nAllow: /app/sign-in\nAllow: /app/sign-up\nAllow: /docs/api/fern-docs/og\nDisallow: /admin\nDisallow: /app/redirect-external/\nDisallow: /legal/\nDisallow: /hog/\nDisallow: /dachshund/\nDisallow: /feature-flags/\nDisallow: /app/sign-in?\nDisallow: /app/sign-up?\nDisallow: */abtest/\nDisallow: */p/\nDisallow: */ab/\nDisallow: /sentry-monitoring\nDisallow: /next/\nDisallow: */page-template-preview/\nDisallow: /v1/\nDisallow: /nl$\nDisallow: /nl/\nAllow: /app/flows\nDisallow: /app/agents\nDisallow: /app/api\nDisallow: /app/developers\nDisallow: /app/files\nDisallow: /app/settings\nDisallow: /app/subscription\n\nUser-Agent: *\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\nAllow: /\nAllow: /share/\nAllow: /embed/\nAllow: /app/sign-in\nAllow: /app/sign-up\nAllow: /docs/api/fern-docs/og\nDisallow: /admin\nDisallow: /app/redirect-external/\nDisallow: /legal/\nDisallow: /hog/\nDisallow: /dachshund/\nDisallow: /feature-flags/\nDisallow: /app/sign-in?\nDisallow: /app/sign-up?\nDisallow: */abtest/\nDisallow: */p/\nDisallow: */ab/\nDisallow: /sentry-monitoring\nDisallow: /next/\nDisallow: */page-template-preview/\nDisallow: /v1/\nDisallow: /nl$\nDisallow: /nl/\nDisallow: /docs/api/\nDisallow: /app/\n\nSitemap: https://elevenlabs.io/sitemap.xml\nSitemap: https://help.elevenlabs.io/hc/sitemap.xml\nSitemap: https://elevenlabs.io/docs/sitemap.xml\nSitemap: https://showcase.elevenlabs.io/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://elevenlabs.io/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 11314, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://elevenlabs.io/sitemap.xml", "contentType": "application/xml; charset=utf-8", "bytes": 24980, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "suno", "name": "Suno", "url": "https://suno.com", "category": "Audio", "notable": true, "checkedAt": "2026-10-14T01:26:13.783Z", "result": { "ok": true, "inputUrl": "https://suno.com/", "finalUrl": "https://suno.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1772, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://suno.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://suno.com/ · HTTP 200 · 267856 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1067 words in the raw HTML, ignoring script and style. H1: “Make any song you can imagine”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "A price is present in structured data (USD 0, USD 10, USD 96, USD 30) but not in the visible text.", "fix": "Put the price in the visible HTML as well. Some crawlers never read JSON-LD." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: WebSite, Organization, PostalAddress, SoftwareApplication, Offer, FAQPage, Question, Answer.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 11816 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://suno.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 267856, "words": 1067, "h1": "Make any song you can imagine", "title": "Suno | AI Music Generator", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [ "USD 0", "USD 10", "USD 96", "USD 30" ], "ldTypes": [ "WebSite", "Organization", "PostalAddress", "SoftwareApplication", "Offer", "FAQPage", "Question", "Answer" ], "ldParsed": 4, "ldBlocks": 4, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://suno.com/robots.txt.", "status": 200, "finalUrl": "https://suno.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 427, "sitemapUrls": [ "https://suno.com/sitemap.xml", "https://suno.com/public-site-sitemap/sitemap.xml", "https://suno.com/music-library-sitemap/sitemap.xml", "https://suno.com/products-sitemap/sitemap.xml", "https://suno.com/tools-sitemap/sitemap.xml", "https://blog.suno.com/sitemap.xml" ], "textLength": 427, "text": "User-Agent: *\nAllow: /\nDisallow: /profile/*/followers\nDisallow: /profile/*/following\nDisallow: /login\nDisallow: /voice/\n\nSitemap: https://suno.com/sitemap.xml\nSitemap: https://suno.com/public-site-sitemap/sitemap.xml\nSitemap: https://suno.com/music-library-sitemap/sitemap.xml\nSitemap: https://suno.com/products-sitemap/sitemap.xml\nSitemap: https://suno.com/tools-sitemap/sitemap.xml\nSitemap: https://blog.suno.com/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://suno.com/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 11816, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://suno.com/sitemap.xml", "contentType": "application/xml", "bytes": 133161, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "udio", "name": "Udio", "url": "https://www.udio.com", "category": "Audio", "notable": false, "checkedAt": "2026-10-14T01:26:16.559Z", "result": { "ok": true, "inputUrl": "https://www.udio.com/", "finalUrl": "https://www.udio.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1905, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.udio.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.udio.com/ · HTTP 200 · 308064 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The response looks like a bot challenge (Udio | AI Music Generator - Official Website), not the page. AI crawlers may hit the same wall.", "fix": "Let the search crawlers through your firewall or CDN, or serve the HTML without a JavaScript challenge." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.udio.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 308064, "words": 8, "h1": "", "title": "Udio | AI Music Generator - Official Website", "challenge": true, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.udio.com/robots.txt.", "status": 200, "finalUrl": "https://www.udio.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 533, "sitemapUrls": [ "https://www.udio.com/sitemap.xml" ], "textLength": 533, "text": "User-Agent: Twitterbot\nAllow: /\nDisallow: /*?\nDisallow: /api/\n\nUser-Agent: Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)\nAllow: /\nDisallow: /*?\nDisallow: /api/\n\nUser-Agent: Slack-ImgProxy 0.19 (+https://api.slack.com/robots)\nAllow: /\nDisallow: /*?\nDisallow: /api/\n\nUser-Agent: Slackbot 1.0 (+https://api.slack.com/robots)\nAllow: /\nDisallow: /*?\nDisallow: /api/\n\nUser-Agent: Googlebot\nAllow: /\nDisallow: /*?\nDisallow: /api/\n\nUser-Agent: *\nAllow: /\nDisallow: /*?\nDisallow: /api/\n\nSitemap: https://www.udio.com/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.udio.com/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 124466, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.udio.com/sitemap.xml", "contentType": "application/xml", "bytes": 1101, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "synthesia", "name": "Synthesia", "url": "https://www.synthesia.io", "category": "Video", "notable": false, "checkedAt": "2026-10-14T01:26:19.470Z", "result": { "ok": true, "inputUrl": "https://www.synthesia.io/", "finalUrl": "https://www.synthesia.io/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1814, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.synthesia.io/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.synthesia.io/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "2818 words in the raw HTML, ignoring script and style. H1: “AI Video and Avatar platform for business”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $56.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 13218 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.synthesia.io/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 524288, "words": 2818, "h1": "AI Video and Avatar platform for business", "title": "Synthesia: #1 AI Video Platform for Business", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [ "$56" ], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.synthesia.io/robots.txt.", "status": 200, "finalUrl": "https://www.synthesia.io/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 183, "sitemapUrls": [ "https://www.synthesia.io/sitemap.xml" ], "textLength": 183, "text": "User-agent: YisouSpider\nDisallow: /\n\nUser-agent: Sogou web spider\nDisallow: /\n\nUser-agent: 360Spider\nDisallow: /\n\nUser-agent: *\nAllow: /\n\nSitemap: https://www.synthesia.io/sitemap.xml" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.synthesia.io/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 13218, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.synthesia.io/sitemap.xml", "contentType": "application/rss+xml; charset=utf-8", "bytes": 73561, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "heygen", "name": "HeyGen", "url": "https://www.heygen.com", "category": "Video", "notable": false, "checkedAt": "2026-10-14T01:26:22.288Z", "result": { "ok": true, "inputUrl": "https://www.heygen.com/", "finalUrl": "https://www.heygen.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 2023, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.heygen.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: GPTBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: OAI-SearchBot.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ChatGPT-User. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ClaudeBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: PerplexityBot.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Google-Extended. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Applebot-Extended. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: CCBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.heygen.com/ · HTTP 200 · 445264 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "2139 words in the raw HTML, ignoring script and style. H1: “AI videos starring you, made in minutes”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $200.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: FAQPage, Question, Answer.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 8254 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.heygen.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 445264, "words": 2139, "h1": "AI videos starring you, made in minutes", "title": "HeyGen: Create AI Videos with Your Personal Avatar", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [ "$200" ], "metaPrices": [], "ldPrices": [], "ldTypes": [ "FAQPage", "Question", "Answer" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.heygen.com/robots.txt.", "status": 200, "finalUrl": "https://www.heygen.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 2999, "sitemapUrls": [ "https://www.heygen.com/sitemap.xml", "https://www.heygen.com/sitemap-blogs.xml", "https://www.heygen.com/sitemap-pt-br.xml", "https://www.heygen.com/sitemap-es-es.xml", "https://www.heygen.com/sitemap-id-id.xml", "https://www.heygen.com/sitemap-fr-fr.xml", "https://www.heygen.com/sitemap-de-de.xml", "https://www.heygen.com/sitemap-ko-kr.xml", "https://www.heygen.com/sitemap-it-it.xml", "https://www.heygen.com/sitemap-vi-vn.xml", "https://www.heygen.com/sitemap-nl-nl.xml", "https://www.heygen.com/sitemap-ja-jp.xml", "https://www.heygen.com/sitemap-tl-ph.xml", "https://www.heygen.com/sitemap-pl-pl.xml", "https://www.heygen.com/sitemap-en-gb.xml", "https://www.heygen.com/sitemap-en-in.xml", "https://www.heygen.com/sitemap-ur-pk.xml", "https://www.heygen.com/sitemap-es-mx.xml", "https://www.heygen.com/sitemap-ar-sa.xml", "https://www.heygen.com/sitemap-tr-tr.xml", "https://www.heygen.com/sitemap-he-il.xml", "https://www.heygen.com/sitemap-uk-ua.xml", "https://www.heygen.com/sitemap-en-ca.xml", "https://www.heygen.com/sitemap-ar-eg.xml", "https://www.heygen.com/sitemap-th-th.xml", "https://www.heygen.com/sitemap-sv-se.xml", "https://www.heygen.com/sitemap-es-ar.xml", "https://www.heygen.com/sitemap-es-co.xml", "https://www.heygen.com/sitemap-zh-tw.xml", "https://www.heygen.com/sitemap-zh-hk.xml" ], "textLength": 2999, "text": "User-agent: *\nDisallow: /ebook/register\nDisallow: /app2/tr\n\nUser-agent: Googlebot\nUser-agent: Googlebot-Image\nUser-agent: Googlebot-Video\nUser-agent: Googlebot-News\nUser-agent: Bingbot\nUser-agent: DuckDuckBot\nUser-agent: DuckAssistBot\nUser-agent: Baiduspider\nUser-agent: YandexBot\nUser-agent: Sogou\nUser-agent: YouBot\nUser-agent: PetalBot\nDisallow: /ebook/register\nDisallow: /app2/\nDisallow: /metrics?\n\nUser-agent: Yeti\nUser-agent: NaverBot\nUser-agent: Daum\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Google-Agent\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: OAI-Operator\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-Code\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Perplexity-User\nAllow: /\n\nUser-agent: Gemini-Fetch\nAllow: /\n\nUser-agent: Gemini-Deep-Research\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Applebot-Extended\nAllow: /\n\nUser-agent: Amazonbot\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: Facebot\nAllow: /\n\nUser-agent: meta-webindexer\nAllow: /\n\nUser-agent: meta-externalfetcher\nAllow: /\n\nUser-agent: meta-externalagent\nAllow: /\n\nUser-agent: AhrefsBot\nAllow: /\n\nUser-agent: AhrefsSiteAudit\nAllow: /\n\nUser-agent: CCBot\nAllow: /\n\nUser-agent: NeevaBot\nUser-agent: PhindBot\nUser-agent: PoeBot\nUser-agent: AndiBot\nUser-agent: komo-crawler\nDisallow: /\n\nSitemap: https://www.heygen.com/sitemap.xml\nSitemap: https://www.heygen.com/sitemap-blogs.xml\nSitemap: https://www.heygen.com/sitemap-pt-br.xml\nSitemap: https://www.heygen.com/sitemap-es-es.xml\nSitemap: https://www.heygen.com/sitemap-id-id.xml\nSitemap: https://www.heygen.com/sitemap-fr-fr.xml\nSitemap: https://www.heygen.com/sitemap-de-de.xml\nSitemap: https://www.heygen.com/sitemap-ko-kr.xml\nSitemap: https://www.heygen.com/sitemap-it-it.xml\nSitemap: https://www.heygen.com/sitemap-vi-vn.xml\nSitemap: https://www.heygen.com/sitemap-nl-nl.xml\nSitemap: https://www.heygen.com/sitemap-ja-jp.xml\nSitemap: https://www.heygen.com/sitemap-tl-ph.xml\nSitemap: https://www.heygen.com/sitemap-pl-pl.xml\nSitemap: https://www.heygen.com/sitemap-en-gb.xml\nSitemap: https://www.heygen.com/sitemap-en-in.xml\nSitemap: https://www.heygen.com/sitemap-ur-pk.xml\nSitemap: https://www.heygen.com/sitemap-es-mx.xml\nSitemap: https://www.heygen.com/sitemap-ar-sa.xml\nSitemap: https://www.heygen.com/sitemap-tr-tr.xml\nSitemap: https://www.heygen.com/sitemap-he-il.xml\nSitemap: https://www.heygen.com/sitemap-uk-ua.xml\nSitemap: https://www.heygen.com/sitemap-en-ca.xml\nSitemap: https://www.heygen.com/sitemap-ar-eg.xml\nSitemap: https://www.heygen.com/sitemap-th-th.xml\nSitemap: https://www.heygen.com/sitemap-sv-se.xml\nSitemap: https://www.heygen.com/sitemap-es-ar.xml\nSitemap: https://www.heygen.com/sitemap-es-co.xml\nSitemap: https://www.heygen.com/sitemap-zh-tw.xml\nSitemap: https://www.heygen.com/sitemap-zh-hk.xml" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "GPTBot", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "OAI-SearchBot", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ChatGPT-User", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ClaudeBot", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "PerplexityBot", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Google-Extended", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Applebot-Extended", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "CCBot", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.heygen.com/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 8254, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.heygen.com/sitemap.xml", "contentType": "application/xml", "bytes": 68791, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "descript", "name": "Descript", "url": "https://www.descript.com", "category": "Audio", "notable": false, "checkedAt": "2026-10-14T01:26:25.315Z", "result": { "ok": true, "inputUrl": "https://www.descript.com/", "finalUrl": "https://www.descript.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 2329, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.descript.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: OAI-SearchBot.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ChatGPT-User. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ClaudeBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: PerplexityBot.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "warn", "detail": "Blocked for / by User-agent: ByteSpider (Disallow: /).", "fix": "Leave the block if you don't want training use. To be cited in AI answers, allow the search crawlers (OAI-SearchBot, PerplexityBot)." }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.descript.com/ · HTTP 200 · 286641 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1750 words in the raw HTML, ignoring script and style. H1: “AI Video Editor”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: BreadcrumbList, ListItem, Organization, FAQPage, Question, Answer.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 6329 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.descript.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 286641, "words": 1750, "h1": "AI Video Editor", "title": "Descript: AI Video & Audio Text-based Editor — Try Free", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "BreadcrumbList", "ListItem", "Organization", "FAQPage", "Question", "Answer" ], "ldParsed": 3, "ldBlocks": 3, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.descript.com/robots.txt.", "status": 200, "finalUrl": "https://www.descript.com/robots.txt", "contentType": "text/plain", "bytes": 751, "sitemapUrls": [ "https://www.descript.com/sitemap.xml" ], "textLength": 751, "text": "Sitemap: https://www.descript.com/sitemap.xml\n\n# Allow AI Search Crawlers\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\n# Ensure Googlebot continues to have access\nUser-agent: Googlebot\nAllow: /\n\n# Block these crawlers entirely\nUser-agent: Amazonbot\nDisallow: /\nUser-agent: ByteSpider\nDisallow: /\nUser-agent: ByteSpiderVideo\nDisallow: /\nUser-agent: ByteSpider-Image\nDisallow: /\n\n# ---------- rules for everyone else ----------\nUser-agent: *\nDisallow: /*?*utm_*\nDisallow: /*?*gclid=*\nDisallow: /*?*fbclid=*\nDisallow: /*?*_page=*\nDisallow: /*features?*_*page=\nDisallow: /blog/author/*?*_page=\nDisallow: /blog/category/*?*_page=\nDisallow: /download/other-device" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "OAI-SearchBot", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ChatGPT-User", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ClaudeBot", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "PerplexityBot", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": false, "countedAsBlock": true, "specific": true, "agent": "ByteSpider", "rule": "Disallow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.descript.com/llms.txt", "contentType": "text/plain", "bytes": 6329, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.descript.com/sitemap.xml", "contentType": "application/xml", "bytes": 265, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "murf", "name": "Murf", "url": "https://murf.ai", "category": "Audio", "notable": false, "checkedAt": "2026-10-14T01:26:28.649Z", "result": { "ok": true, "inputUrl": "https://murf.ai/", "finalUrl": "https://murf.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1615, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://murf.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://murf.ai/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1760 words in the raw HTML, ignoring script and style. H1: “Build AI Voice Experiences That Are Natural, Scalable & Efficient”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "warn", "detail": "JSON-LD types: WebPage, WebSite, Corporation, Person, ContactPoint, Thing.", "fix": "Add SoftwareApplication or Product JSON-LD with an Offer so the name, description and price are explicit." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 9403 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://murf.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 524288, "words": 1760, "h1": "Build AI Voice Experiences That Are Natural, Scalable & Efficient", "title": "Leading AI Voice & Conversational Agent Platform - Murf AI", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "WebPage", "WebSite", "Corporation", "Person", "ContactPoint", "Thing" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://murf.ai/robots.txt.", "status": 200, "finalUrl": "https://murf.ai/robots.txt", "contentType": "text/plain", "bytes": 355, "sitemapUrls": [ "https://murf.ai/public-assets/sitemaps/sitemap.xml", "https://murf.ai/public-assets/sitemaps/translated-page-sitemap.xml", "https://murf.ai/public-assets/sitemaps/helpsitemap.xml" ], "textLength": 355, "text": "# https://www.robotstxt.org/robotstxt.html\nUser-agent: *\nDisallow: /swagger.json\nDisallow: /share/\nDisallow: /Share/\nDisallow: /Shared/\nDisallow: /shared/\nSitemap: https://murf.ai/public-assets/sitemaps/sitemap.xml\nSitemap: https://murf.ai/public-assets/sitemaps/translated-page-sitemap.xml\nSitemap: https://murf.ai/public-assets/sitemaps/helpsitemap.xml " }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://murf.ai/llms.txt", "contentType": "text/plain", "bytes": 9403, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://murf.ai/sitemap.xml", "contentType": "application/rss+xml; charset=utf-8", "bytes": 36204, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "speechify", "name": "Speechify", "url": "https://speechify.com", "category": "Audio", "notable": false, "checkedAt": "2026-10-14T01:26:31.270Z", "result": { "ok": true, "inputUrl": "https://speechify.com/", "finalUrl": "https://speechify.com/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 1956, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://speechify.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: GPTBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: OAI-SearchBot.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ChatGPT-User. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ClaudeBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://speechify.com/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "2974 words in the raw HTML, ignoring script and style. H1: “Speechify , Text to Speech . Voice Typing . Fast Answers .”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 24196 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://speechify.com/sitemap_index.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 524288, "words": 2974, "h1": "Speechify , Text to Speech . Voice Typing . Fast Answers .", "title": "Speechify: Text to Speech & Voice Typing AI Assistant | 60M+ Users", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://speechify.com/robots.txt.", "status": 200, "finalUrl": "https://speechify.com/robots.txt", "contentType": "text/plain", "bytes": 389, "sitemapUrls": [ "https://speechify.com/sitemap_index.xml" ], "textLength": 389, "text": "User-Agent: *\nAllow: /onboarding/sign-in/\nDisallow: /*?landing=\nDisallow: /api/\nDisallow: /onboarding/\n\nUser-Agent: AdsBot-Google\nDisallow: /*?landing=\nDisallow: /api/\n\nUser-Agent: OAI-SearchBot\nUser-Agent: ChatGPT-User\nUser-Agent: GPTBot\nUser-Agent: ClaudeBot\nUser-Agent: Claude-User\nUser-Agent: Claude-SearchBot\nAllow: /\nDisallow: /api/\n\nSitemap: https://speechify.com/sitemap_index.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "GPTBot", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "OAI-SearchBot", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ChatGPT-User", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ClaudeBot", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://speechify.com/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 24196, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://speechify.com/sitemap_index.xml", "contentType": "application/xml", "bytes": 12316, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "jasper", "name": "Jasper", "url": "https://www.jasper.ai", "category": "Writing", "notable": true, "checkedAt": "2026-10-14T01:26:34.229Z", "result": { "ok": true, "inputUrl": "https://www.jasper.ai/", "finalUrl": "https://www.jasper.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1789, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.jasper.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: GPTBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: OAI-SearchBot.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ChatGPT-User. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: ClaudeBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: PerplexityBot.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Google-Extended. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Applebot-Extended. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: CCBot. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: Bytespider. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.jasper.ai/ · HTTP 200 · 486145 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "2501 words in the raw HTML, ignoring script and style. H1: “Put AI agents to work for marketing”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "A price is present in structured data (USD 0) but not in the visible text.", "fix": "Put the price in the visible HTML as well. Some crawlers never read JSON-LD." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, ImageObject, ContactPoint, Thing, WebSite, FAQPage, Question, Answer, SoftwareApplication, Offer, AggregateRating.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 12237 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.jasper.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 486145, "words": 2501, "h1": "Put AI agents to work for marketing", "title": "Put AI agents to work for marketing | Jasper", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [ "USD 0" ], "ldTypes": [ "Organization", "ImageObject", "ContactPoint", "Thing", "WebSite", "FAQPage", "Question", "Answer", "SoftwareApplication", "Offer", "AggregateRating" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.jasper.ai/robots.txt.", "status": 200, "finalUrl": "https://www.jasper.ai/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 665, "sitemapUrls": [ "https://www.jasper.ai/sitemap.xml" ], "textLength": 665, "text": "User-agent: Googlebot\nUser-agent: Bingbot\nUser-agent: Slurp\nAllow: /\n\nUser-agent: PerplexityBot\nUser-agent: YouBot\nUser-agent: Applebot-Extended\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: anthropic-ai\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: CCBot\nUser-agent: cohere-ai\nUser-agent: AI2Bot\nUser-agent: Diffbot\nUser-agent: Bytespider\nUser-agent: meta-externalagent\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: Omgilibot\nDisallow: /\n\nUser-agent: *\nDisallow: /credits-rate-card\n\nContent-Signal: ai-train=yes, search=yes, ai-input=yes\n\nSitemap: https://www.jasper.ai/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "GPTBot", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "OAI-SearchBot", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ChatGPT-User", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "ClaudeBot", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "PerplexityBot", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Google-Extended", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Applebot-Extended", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "CCBot", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": true, "agent": "Bytespider", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.jasper.ai/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 12237, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.jasper.ai/sitemap.xml", "contentType": "application/rss+xml; charset=utf-8", "bytes": 123127, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "copyai", "name": "Copy.ai", "url": "https://www.copy.ai", "category": "Writing", "notable": false, "checkedAt": "2026-10-14T01:26:37.025Z", "result": { "ok": true, "inputUrl": "https://www.copy.ai/", "finalUrl": "https://www.copy.ai/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1696, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.copy.ai/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.copy.ai/ · HTTP 200 · 212869 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1245 words in the raw HTML, ignoring script and style. H1: “GTM Platform”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $16.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.copy.ai/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 212869, "words": 1245, "h1": "GTM Platform", "title": "Future proof your business with GTM AI", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [ "$16" ], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.copy.ai/robots.txt.", "status": 200, "finalUrl": "https://www.copy.ai/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 854, "sitemapUrls": [ "https://www.copy.ai/sitemap.xml", "https://www.copy.ai/sitemap.xml" ], "textLength": 850, "text": "User-agent: *\nAllow: /\n\n# 🚫 Block paginated URLs (common Webflow blog query params)\nDisallow: /*?88f8398a_page=\nDisallow: /*?4306d73d_page=\nDisallow: /*?4e25f633_page=\nDisallow: /*?_page=\nDisallow: /*&ec3b096a_page=\nDisallow: /*&fbc21e0b_page=\nDisallow: /*?*page=\n\n# 🚫 Block internal utility pages or duplicated content sections\nDisallow: *vendors?q\nDisallow: /components-for-blog\nDisallow: /prompt-library-category/\nDisallow: /tool-categories/\nDisallow: /prompt-library-use-case/\nDisallow: /glossary-categories/\nDisallow: /glossary\nDisallow: /tag\nDisallow: /for\nDisallow: /archive/\nDisallow: /alternatives\nDisallow: /workflow-template\nDisallow: /abm/\nDisallow: /abm-templates\nDisallow: /author/\nDisallow: /app/\nDisallow: /community/\nDisallow: /learn/certificates/\n\nSitemap: https://www.copy.ai/sitemap.xml\n\nSitemap: https://www.copy.ai/sitemap.xml" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.copy.ai/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 22384, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.copy.ai/sitemap.xml", "contentType": "application/rss+xml; charset=utf-8", "bytes": 507942, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "writesonic", "name": "Writesonic", "url": "https://writesonic.com", "category": "Writing", "notable": false, "checkedAt": "2026-10-14T01:26:39.727Z", "result": { "ok": true, "inputUrl": "https://writesonic.com/", "finalUrl": "https://writesonic.com/", "checkedPath": "/", "pageStatus": 200, "truncated": true, "elapsedMs": 2146, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://writesonic.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://writesonic.com/ · HTTP 200 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "2372 words in the raw HTML, ignoring script and style. H1: “Win customers from AI search.”. Counted from the first 512 KB.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $350, $150, $1.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, PostalAddress, ContactPoint, WebSite, SoftwareApplication, AggregateRating.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 11720 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://writesonic.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 524288, "words": 2372, "h1": "Win customers from AI search.", "title": "Writesonic | The AI Search Growth Engine. Win Customers.", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [ "$350", "$150", "$1" ], "metaPrices": [], "ldPrices": [], "ldTypes": [ "Organization", "PostalAddress", "ContactPoint", "WebSite", "SoftwareApplication", "AggregateRating" ], "ldParsed": 3, "ldBlocks": 3, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://writesonic.com/robots.txt.", "status": 200, "finalUrl": "https://writesonic.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 164, "sitemapUrls": [ "https://writesonic.com/sitemap.xml", "https://writesonic.com/blog/sitemap.xml" ], "textLength": 164, "text": "User-Agent: *\nAllow: /\n\n# Host\nHost: https://writesonic.com/\n\n# Sitemap\nSitemap: https://writesonic.com/sitemap.xml\nSitemap: https://writesonic.com/blog/sitemap.xml" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://writesonic.com/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 11720, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://writesonic.com/sitemap.xml", "contentType": "application/xml; charset=UTF-8", "bytes": 15098, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": true } } } }, { "id": "grammarly", "name": "Grammarly", "url": "https://www.grammarly.com", "category": "Writing", "notable": true, "checkedAt": "2026-10-14T01:26:42.880Z", "result": { "ok": true, "inputUrl": "https://www.grammarly.com/", "finalUrl": "https://www.grammarly.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1797, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.grammarly.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.grammarly.com/ · HTTP 200 · 501654 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1226 words in the raw HTML, ignoring script and style. H1: “Think big. We’ll take care of the details.”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "pass", "detail": "Price-like text in the visible HTML: $0, $12, $30, $5,00.", "fix": "" }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.grammarly.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 501654, "words": 1226, "h1": "Think big. We’ll take care of the details.", "title": "Grammarly: Free AI Writing Assistance", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [ "$0", "$12", "$30", "$5,00" ], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.grammarly.com/robots.txt.", "status": 200, "finalUrl": "https://www.grammarly.com/robots.txt", "contentType": "text/plain; charset=UTF-8", "bytes": 289, "sitemapUrls": [ "https://www.grammarly.com/sitemap.xml", "https://www.grammarly.com/blog/sitemap_index.xml" ], "textLength": 289, "text": "User-agent: *\n\nDisallow: /voucher/\nDisallow: /social/\nDisallow: /api/\nDisallow: /enterprise/\nDisallow: /nonprofits-ngos\nDisallow: /plans/try\nDisallow: /es-us/\nDisallow: /blog/search/\n\nSitemap: https://www.grammarly.com/sitemap.xml\nSitemap: https://www.grammarly.com/blog/sitemap_index.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.grammarly.com/llms.txt", "contentType": "text/html; charset=UTF-8", "bytes": 474099, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.grammarly.com/sitemap.xml", "contentType": "text/xml; charset=UTF-8", "bytes": 1015, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "quillbot", "name": "QuillBot", "url": "https://quillbot.com", "category": "Writing", "notable": false, "checkedAt": "2026-10-14T01:26:45.684Z", "result": { "ok": true, "inputUrl": "https://quillbot.com/", "finalUrl": "https://quillbot.com/", "checkedPath": "/", "pageStatus": 403, "truncated": false, "elapsedMs": 1648, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "HTTP 403", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "warn", "detail": "GPTBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "warn", "detail": "OAI-SearchBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "warn", "detail": "ChatGPT-User: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "warn", "detail": "ClaudeBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "warn", "detail": "PerplexityBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "warn", "detail": "Google-Extended: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "warn", "detail": "Applebot-Extended: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "warn", "detail": "CCBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "warn", "detail": "Bytespider: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "Couldn't read robots.txt. xAI has not published a crawler user-agent, so there is no token that reliably allows or blocks Grok.", "fix": "Keep the page's facts in the raw HTML. robots.txt cannot reliably control Grok until xAI publishes a token." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://quillbot.com/ · HTTP 403 · 5818 bytes. JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 9 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 403.", "fix": "Serve /llms.txt as a public plain-text file with HTTP 200." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 403, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 5818, "words": 9, "h1": "", "title": "Just a moment...", "challenge": true, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "unavailable", "note": "HTTP 403", "status": 403, "finalUrl": "https://quillbot.com/robots.txt", "contentType": "text/html; charset=UTF-8", "bytes": 5891, "sitemapUrls": [], "textLength": 5891, "text": "Just a moment...
" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": null, "countedAsBlock": false, "specific": false, "agent": "", "rule": "" } }, "xai": { "GrokBot": { "allowed": null, "specific": false, "agent": "", "rule": "" }, "xAI-SearchBot": { "allowed": null, "specific": false, "agent": "", "rule": "" } }, "llms": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://quillbot.com/llms.txt", "contentType": "text/html; charset=UTF-8", "bytes": 5885, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 403, "error": "", "finalUrl": "https://quillbot.com/sitemap.xml", "contentType": "text/html; charset=UTF-8", "bytes": 5894, "checkStatus": "warn", "pass": false, "looksLikeSitemap": false, "mentionsPricing": false } } } }, { "id": "wordtune", "name": "Wordtune", "url": "https://www.wordtune.com", "category": "Writing", "notable": false, "checkedAt": "2026-10-14T01:26:48.336Z", "result": { "ok": true, "inputUrl": "https://www.wordtune.com/", "finalUrl": "https://www.wordtune.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1793, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.wordtune.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.wordtune.com/ · HTTP 200 · 124083 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1160 words in the raw HTML, ignoring script and style. H1: “Express yourself with confidence”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.wordtune.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 124083, "words": 1160, "h1": "Express yourself with confidence", "title": "Wordtune — Express yourself with confidence", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.wordtune.com/robots.txt.", "status": 200, "finalUrl": "https://www.wordtune.com/robots.txt", "contentType": "text/plain; charset=utf-8", "bytes": 116, "sitemapUrls": [ "https://www.wordtune.com/sitemap.xml", "https://www.wordtune.com/sitemap.xml" ], "textLength": 116, "text": "User-agent: *\nAllow: /\n\nSitemap: https://www.wordtune.com/sitemap.xml\n\nSitemap: https://www.wordtune.com/sitemap.xml" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.wordtune.com/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 12730, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.wordtune.com/sitemap.xml", "contentType": "application/rss+xml; charset=utf-8", "bytes": 51416, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "deepl", "name": "DeepL", "url": "https://www.deepl.com", "category": "Writing", "notable": true, "checkedAt": "2026-10-14T01:26:51.136Z", "result": { "ok": true, "inputUrl": "https://www.deepl.com/", "finalUrl": "https://www.deepl.com/en", "checkedPath": "/en", "pageStatus": 200, "truncated": false, "elapsedMs": 2369, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /en. Fetched https://www.deepl.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for /en. No disallow matches. Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.deepl.com/en · HTTP 200 · 389680 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "1272 words in the raw HTML, ignoring script and style. H1: “AI solutions that help you get work done”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, ImageObject, Place, QuantitativeValue, ContactPoint, WebSite, ItemList, ListItem, WebApplication, Action, EntryPoint, Offer.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "warn", "detail": "/llms.txt returned HTTP 404.", "fix": "Publish a plain-text /llms.txt with a one-line summary and links to your pricing, docs and FAQ." }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.deepl.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 389680, "words": 1272, "h1": "AI solutions that help you get work done", "title": "DeepL AI Platform: Translation, Voice & API", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [ "Organization", "ImageObject", "Place", "QuantitativeValue", "ContactPoint", "WebSite", "ItemList", "ListItem", "WebApplication", "Action", "EntryPoint", "Offer" ], "ldParsed": 3, "ldBlocks": 3, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.deepl.com/robots.txt.", "status": 200, "finalUrl": "https://www.deepl.com/robots.txt", "contentType": "text/plain; charset=UTF-8", "bytes": 1722, "sitemapUrls": [ "https://www.deepl.com/sitemap.xml" ], "textLength": 1722, "text": "############################################\n# Last full revision: 2026-08-31\n############################################\n\nSitemap: https://www.deepl.com/sitemap.xml\n\n##### Ads & Monetization #####\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nUser-agent: AdIdxBot\nUser-agent: Mediapartners-Google\nUser-agent: Ads-Naver\nUser-agent: Y!J-BRY\nUser-agent: Y!J-BRZ\n# Full access for ads quality assessment & UTM tracking\nAllow: /\n# API legacy docs exclusion\nDisallow: /*docs-api-legacy$\nDisallow: /*docs-api-legacy/\n# Cloudflare paths\nDisallow: /cdn-cgi/bm/cv/\nDisallow: /cdn-cgi/challenge-platform/\nDisallow: /cdn-cgi/images/trace/\nDisallow: /cdn-cgi/rum\nDisallow: /cdn-cgi/scripts/\nDisallow: /cdn-cgi/styles/\nDisallow: /cdn-cgi/zaraz/\n\n##### Organic #####\nUser-agent: Googlebot\nUser-agent: Bingbot\nUser-agent: Yeti\nUser-agent: Y!J-ASR\nUser-agent: Y!J-WSC\nUser-agent: Slurp\n# Block campaign paths & UTM parameters\nDisallow: */translator/c/\nDisallow: /*utm_campaign=*\nDisallow: /*utm_source=linguee*\n# API legacy docs exclusion\nDisallow: /*docs-api-legacy$\nDisallow: /*docs-api-legacy/\n# Cloudflare paths\nDisallow: /cdn-cgi/bm/cv/\nDisallow: /cdn-cgi/challenge-platform/\nDisallow: /cdn-cgi/images/trace/\nDisallow: /cdn-cgi/rum\nDisallow: /cdn-cgi/scripts/\nDisallow: /cdn-cgi/styles/\nDisallow: /cdn-cgi/zaraz/\n\n##### Everone else #####\nUser-agent: *\n# Explicit access for campaign paths\nAllow: */translator/c/\n# API legacy docs exclusion\nDisallow: /*docs-api-legacy$\nDisallow: /*docs-api-legacy/\n# Cloudflare paths\nDisallow: /cdn-cgi/bm/cv/\nDisallow: /cdn-cgi/challenge-platform/\nDisallow: /cdn-cgi/images/trace/\nDisallow: /cdn-cgi/rum\nDisallow: /cdn-cgi/scripts/\nDisallow: /cdn-cgi/styles/\nDisallow: /cdn-cgi/zaraz/" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "" } }, "llms": { "fetched": true, "http": 404, "error": "", "finalUrl": "https://www.deepl.com/llms.txt", "contentType": "text/html; charset=utf-8", "bytes": 63741, "checkStatus": "warn", "pass": false }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.deepl.com/sitemap.xml", "contentType": "application/xml", "bytes": 425, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "notion", "name": "Notion", "url": "https://www.notion.com", "category": "Workspace", "notable": true, "checkedAt": "2026-10-14T01:26:54.508Z", "result": { "ok": true, "inputUrl": "https://www.notion.com/", "finalUrl": "https://www.notion.com/", "checkedPath": "/", "pageStatus": 200, "truncated": false, "elapsedMs": 1938, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "Rules for /. Fetched https://www.notion.com/robots.txt.", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. OpenAI says this user-triggered agent may ignore robots.txt.", "fix": "" }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *.", "fix": "" }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Gemini training; it is not the search crawler.", "fix": "" }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This token only opts out of Apple Intelligence training; it is not the search crawler.", "fix": "" }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "pass", "detail": "Allowed for / (Allow: /). Group: *. This crawler is for training, not AI search.", "fix": "" }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "xAI has not published a crawler user-agent. Reported names in public registries are GrokBot and xAI-SearchBot; xAI has not confirmed either. On this file, GrokBot is allowed only by the default rule; xAI-SearchBot is allowed only by the default rule.", "fix": "robots.txt can't reliably invite Grok until xAI publishes a token. Put the facts you want repeated into the raw HTML." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://www.notion.com/ · HTTP 200 · 210083 bytes. JavaScript was not run.", "checks": [ { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "pass", "detail": "444 words in the raw HTML, ignoring script and style. H1: “Where teams and agents Think together.”.", "fix": "" }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "A price is present in structured data (USD 0) but not in the visible text.", "fix": "Put the price in the visible HTML as well. Some crawlers never read JSON-LD." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "pass", "detail": "JSON-LD types: Organization, WebSite, SoftwareApplication, Offer.", "fix": "" } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 6803 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "pass", "detail": "Found an XML sitemap at https://www.notion.com/sitemap.xml (HTTP 200).", "fix": "" } ] } ], "facts": { "contentType": "text/html; charset=utf-8", "bytes": 210083, "words": 444, "h1": "Where teams and agents Think together.", "title": "The AI workspace that works for you. | Notion", "challenge": false, "isHtml": true, "homepageLinksPricing": true, "visiblePrices": [], "metaPrices": [], "ldPrices": [ "USD 0" ], "ldTypes": [ "Organization", "WebSite", "SoftwareApplication", "Offer" ], "ldParsed": 1, "ldBlocks": 1, "ldInvalid": 0, "robots": { "state": "parsed", "note": "Fetched https://www.notion.com/robots.txt.", "status": 200, "finalUrl": "https://www.notion.com/robots.txt", "contentType": "text/plain", "bytes": 1178, "sitemapUrls": [ "https://www.notion.com/sitemap.xml", "https://sitemaps.notion.com/sitemap-categories.xml", "https://sitemaps.notion.com/sitemap-collections.xml", "https://www.notion.com/sitemap-root.xml", "https://www.notion.com/blog/sitemap.xml", "https://www.notion.com/customers/sitemap.xml", "https://www.notion.com/help/sitemap.xml", "https://www.notion.com/md/sitemap.xml", "https://www.notion.com/releases/sitemap.xml", "https://www.notion.com/use-case/sitemap.xml", "https://www.notion.com/feature/sitemap.xml", "https://www.notion.com/front-static/sitemap.xml" ], "textLength": 1178, "text": "User-Agent: *\nAllow: /\nDisallow: /invite/\nDisallow: /*/invite/\nDisallow: /templates/search?query=*\nDisallow: /*/templates/search?query=*\nDisallow: /lp/webinars/\nDisallow: /lp/webinars/*\nDisallow: /_vercel/insights/view\nDisallow: /embed/*\nDisallow: /*/embed/*\nDisallow: /product/agents-homepage\n\nUser-Agent: BLEXBot\nDisallow: /\n\nUser-Agent: AhrefsBot\nDisallow: /\n\nUser-Agent: Amazonbot\nDisallow: /\n\nUser-Agent: Amzn-SearchBot\nDisallow: /\n\nUser-Agent: meta-externalagent\nDisallow: /\n\nUser-Agent: SemrushBot\nDisallow: /\n\nUser-Agent: dotbot\nDisallow: /\n\nSitemap: https://www.notion.com/sitemap.xml\nSitemap: https://sitemaps.notion.com/sitemap-categories.xml\nSitemap: https://sitemaps.notion.com/sitemap-collections.xml\nSitemap: https://www.notion.com/sitemap-root.xml\nSitemap: https://www.notion.com/blog/sitemap.xml\nSitemap: https://www.notion.com/customers/sitemap.xml\nSitemap: https://www.notion.com/help/sitemap.xml\nSitemap: https://www.notion.com/md/sitemap.xml\nSitemap: https://www.notion.com/releases/sitemap.xml\nSitemap: https://www.notion.com/use-case/sitemap.xml\nSitemap: https://www.notion.com/feature/sitemap.xml\nSitemap: https://www.notion.com/front-static/sitemap.xml\n" }, "crawlers": { "gptbot": { "token": "GPTBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "oai-searchbot": { "token": "OAI-SearchBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "chatgpt-user": { "token": "ChatGPT-User", "kind": "user", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "claudebot": { "token": "ClaudeBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "perplexitybot": { "token": "PerplexityBot", "kind": "search", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "google-extended": { "token": "Google-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "applebot-extended": { "token": "Applebot-Extended", "kind": "optout", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "ccbot": { "token": "CCBot", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" }, "bytespider": { "token": "Bytespider", "kind": "training", "allowed": true, "countedAsBlock": false, "specific": false, "agent": "*", "rule": "Allow: /" } }, "xai": { "GrokBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" }, "xAI-SearchBot": { "allowed": true, "specific": false, "agent": "*", "rule": "Allow: /" } }, "llms": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.notion.com/llms.txt", "contentType": "text/plain; charset=utf-8", "bytes": 6803, "checkStatus": "pass", "pass": true }, "sitemap": { "fetched": true, "http": 200, "error": "", "finalUrl": "https://www.notion.com/sitemap.xml", "contentType": "application/xml", "bytes": 12688, "checkStatus": "pass", "pass": true, "looksLikeSitemap": true, "mentionsPricing": false } } } }, { "id": "gamma", "name": "Gamma", "url": "https://gamma.app", "category": "Workspace", "notable": false, "checkedAt": "2026-10-14T01:26:57.451Z", "result": { "ok": true, "inputUrl": "https://gamma.app/", "finalUrl": "https://gamma.app/", "checkedPath": "/", "pageStatus": 403, "truncated": true, "elapsedMs": 2088, "sections": [ { "id": "robots", "title": "AI crawlers in robots.txt", "note": "HTTP 403", "checks": [ { "id": "gptbot", "name": "GPTBot", "hint": "OpenAI · training", "status": "warn", "detail": "GPTBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "hint": "OpenAI · ChatGPT search", "status": "warn", "detail": "OAI-SearchBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "chatgpt-user", "name": "ChatGPT-User", "hint": "OpenAI · user-triggered fetch", "status": "warn", "detail": "ChatGPT-User: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "claudebot", "name": "ClaudeBot", "hint": "Anthropic · training", "status": "warn", "detail": "ClaudeBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "perplexitybot", "name": "PerplexityBot", "hint": "Perplexity · search", "status": "warn", "detail": "PerplexityBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "google-extended", "name": "Google-Extended", "hint": "Google · Gemini training opt-out", "status": "warn", "detail": "Google-Extended: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "applebot-extended", "name": "Applebot-Extended", "hint": "Apple · Apple Intelligence training opt-out", "status": "warn", "detail": "Applebot-Extended: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "ccbot", "name": "CCBot", "hint": "Common Crawl · training corpus", "status": "warn", "detail": "CCBot: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "bytespider", "name": "Bytespider", "hint": "ByteDance · training", "status": "warn", "detail": "Bytespider: robots.txt could not be used (HTTP 403). This is not counted as a disallow.", "fix": "Serve a public /robots.txt that returns HTTP 200 and plain text." }, { "id": "xai-grok", "name": "xAI / Grok", "hint": "No official user-agent published", "status": "warn", "detail": "Couldn't read robots.txt. xAI has not published a crawler user-agent, so there is no token that reliably allows or blocks Grok.", "fix": "Keep the page's facts in the raw HTML. robots.txt cannot reliably control Grok until xAI publishes a token." } ] }, { "id": "html", "title": "Raw HTML, without JavaScript", "note": "Fetched https://gamma.app/ · HTTP 403 · 524288 bytes (cut off at 512 KB). JavaScript was not run.", "checks": [ { "id": "page-http", "name": "Page response", "hint": "HTTP status", "status": "warn", "detail": "The URL returned HTTP 403.", "fix": "Open the exact URL visitors and crawlers should read. This score is for the response this address returned." }, { "id": "visible-text", "name": "Visible text without JavaScript", "hint": "Script and style ignored", "status": "fail", "detail": "The URL returned HTTP 403 and only 9 visible words. This is not the page content.", "fix": "Check the exact public URL. A 4xx or 5xx response is what a crawler may keep." }, { "id": "pricing", "name": "Pricing in the raw HTML", "hint": "Visible text", "status": "warn", "detail": "No price-like text on this URL.", "fix": "If prices live on another page, check that URL too, and keep the numbers in the HTML rather than loading them with JavaScript." }, { "id": "jsonld", "name": "JSON-LD", "hint": "Types in the raw HTML", "status": "fail", "detail": "No JSON-LD script in the HTML.", "fix": "Add a script type=\"application/ld+json\" with Organization and SoftwareApplication or Product, including the name and an Offer." } ] }, { "id": "files", "title": "llms.txt and sitemap.xml", "note": "", "checks": [ { "id": "llms", "name": "llms.txt", "hint": "Same host", "status": "pass", "detail": "Found /llms.txt (HTTP 200, 4328 bytes).", "fix": "" }, { "id": "sitemap", "name": "sitemap.xml", "hint": "Same host", "status": "warn", "detail": "/sitemap.xml returned HTTP 403, and no usable sitemap was listed in robots.txt.", "fix": "Publish an XML sitemap at /sitemap.xml, or point a Sitemap line in robots.txt at one, listing your key pages." } ] } ], "facts": { "contentType": "text/html; charset=UTF-8", "bytes": 524288, "words": 9, "h1": "", "title": "", "challenge": false, "isHtml": true, "homepageLinksPricing": false, "visiblePrices": [], "metaPrices": [], "ldPrices": [], "ldTypes": [], "ldParsed": 0, "ldBlocks": 0, "ldInvalid": 0, "robots": { "state": "unavailable", "note": "HTTP 403", "status": 403, "finalUrl": "https://gamma.app/robots.txt", "contentType": "text/html; charset=UTF-8", "bytes": 512000, "sitemapUrls": [], "textLength": 512000, "text": "\n\n \n \n \n \n \n \n \n \n