{
  "acceptance": "Control-file availability, page availability and crawler behavior each have their own evidence record.",
  "automated_checks": true,
  "built_at": "2026-11-02T17:45:12.230789+00:00",
  "description": "A robots.txt retrieval failure can affect crawling differently from one failed page. Diagnose them separately.",
  "diagram": [
    "Check control file",
    "Compare page failure",
    "Inspect crawl evidence",
    "Verify later behavior"
  ],
  "evidence": {
    "observed_at": "2026-11-02T16:10:47+00:00",
    "period": {
      "end": "2026-11-02T16:10:47+00:00",
      "start": "2026-10-31T15:00:00+00:00",
      "timezone": "UTC"
    },
    "rows": [
      {
        "label": "HTTP 200",
        "unit": "requests",
        "value": 616
      },
      {
        "label": "HTTP 502",
        "unit": "requests",
        "value": 45
      }
    ],
    "scope": "own_site_selected_public_paths",
    "self_tests_excluded": false
  },
  "figure_caption": "Original explanatory workflow. Numeric callouts reproduce dated own-site evidence; they are not an AI platform test.",
  "figure_title": "Separate robots.txt failures from page errors",
  "group": "Access diagnostics",
  "human_review_claim": false,
  "intent": "robots-file-errors-vs-page-errors",
  "interpretation": "Google documents special handling for robots.txt server errors. A page-level 502 does not establish that the control file failed or that Google stopped crawling the site.",
  "keyword_volume": "Not obtained",
  "limits": "This guide does not diagnose a robots.txt failure from sample page errors. Consult the current official specification.",
  "path": "/ai-analytics/guides/robots-file-errors-vs-page-errors/",
  "policy": "analytics-evidence-v1",
  "problem": "A robots.txt retrieval failure can affect crawling differently from one failed page. Diagnose them separately.",
  "product": "/ai-analytics",
  "published_at": "2026-11-03T00:15:06.258218+00:00",
  "reader": "Website owners, SEO analysts and site operators",
  "related": [
    "/learn/ai-crawler-logs-vs-ai-visibility/",
    "/research/ai-crawler-analytics/"
  ],
  "slug": "robots-file-errors-vs-page-errors",
  "source_signature": "84637a2a46e2c8ecc66665ae803b5225b74167337aa3e437fcc20c3320df7c54",
  "sources": [
    {
      "hash": "a58f4e30eff99d5e8795675eae2e9290faef585815555db569b1814eedf66d6f",
      "locator": "status",
      "observed_at": "2026-11-02T16:10:47+00:00",
      "title": "Saved own-site monitoring evidence",
      "url": "https://future-intech.com/research/ai-crawler-analytics/"
    },
    {
      "hash": "fb7afe5770552b183375179e5f0895802315898af540e07c7b28f96bda323a74",
      "locator": "Handling of errors and HTTP status codes",
      "observed_at": "2026-11-02T17:45:12.318777+00:00",
      "title": "Google robots.txt specification",
      "url": "https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec"
    }
  ],
  "sources_required": [
    "google_robots"
  ],
  "steps": [
    "Retrieve the actual robots.txt response under normal permitted access and retain time, status and body.",
    "Compare its evidence with the page failure window without assuming a shared cause.",
    "Inspect available crawl information separately, preserving its native dates and scope.",
    "After a verified control-file fix, check its deployed response and later crawl behavior."
  ],
  "title": "Separate robots.txt failures from page errors",
  "worked_example": "The status table alone cannot establish a robots.txt incident. It identifies a request failure population for which more location-specific evidence is needed."
}