{
  "acceptance": "The investigation can trace each conclusion to the dated rule, request and response. A later policy edit does not silently change the interpretation of old evidence.",
  "automated_checks": true,
  "batch_id": "analytics-editorial-20261104-v1",
  "built_at": "2026-11-04T11:50:28.566595+00:00",
  "claims": [
    {
      "id": "c1",
      "section": "Explanation and verification steps",
      "source_numbers": [
        1
      ],
      "statement": "The violations view identifies crawler requests to explicitly disallowed paths."
    },
    {
      "id": "c2",
      "section": "Explanation and verification steps",
      "source_numbers": [
        1
      ],
      "statement": "The view identifies the directive line used for the path."
    },
    {
      "id": "c3",
      "section": "Explanation and verification steps",
      "source_numbers": [
        1
      ],
      "statement": "The count applies to the disallowed path/directive pair in the selected interval."
    },
    {
      "id": "c4",
      "section": "Explanation and verification steps",
      "source_numbers": [
        1
      ],
      "statement": "A path-specific WAF rule is a documented enforcement option."
    }
  ],
  "description": "Use a dated rule, requested path and response together when investigating crawler requests to a disallowed URL.",
  "diagram": [
    "Dated robots directive",
    "Requested path and classification",
    "Observed HTTP response",
    "Separate enforcement decision"
  ],
  "draft_hash": "edb97c24feeef549940ab0d01cd38c31bd442b2fb0541b13ebae7155d4ea34b4",
  "evidence": {
    "observed_at": "2026-11-02T17:31:04.468187+00:00",
    "period": {
      "end": "2026-11-02T17:31:04.468187+00:00",
      "start": "2026-11-02T17:31:04.468187+00:00",
      "timezone": "UTC"
    },
    "rows": [],
    "scope": "Official documentation; no customer or own-site traffic measurement"
  },
  "evidence_kind": "documentation",
  "figure_caption": "Original explanatory diagram based on the dated sources below. Any example values are hypothetical; this is not a screenshot, customer measurement or AI platform result.",
  "figure_kind": "comparison",
  "figure_notes": [
    "Use the version applicable to the observation interval.",
    "Identify the exact path and classification evidence.",
    "A request to a disallowed path is not proof of delivery.",
    "Test HTTP access controls separately from robots directives."
  ],
  "figure_title": "Separate robots directives from observed access enforcement",
  "group": "Crawler access and indexing",
  "human_review_claim": false,
  "intent": "robots-directives-vs-access-enforcement",
  "interpretation": "Cloudflare describes its violations view as identifying requests to paths explicitly disallowed by robots.txt. The documented record includes the path, directive line and request count for the selected interval. That is useful evidence of a request relative to a rule, but it is a different question from whether an HTTP security rule blocked the request. Keep the published directive, crawler classification and delivered response as separate observations.",
  "keyword_volume": "Not obtained",
  "limits": "This is a technical evidence workflow, not a legal conclusion about permitted use. It does not establish that a model trained on, recommended or cited the content.",
  "paid_api_calls": 0,
  "path": "/ai-analytics/guides/robots-directives-vs-access-enforcement/",
  "policy": "session-evidence-editorial-v1",
  "preparation_mode": "session_editorial_batch",
  "problem": "A robots-related report labels a request as blocked or compliant without retaining the applicable directive and the actual response.",
  "product": "/ai-analytics",
  "published_at": "2026-11-05T09:00:06.127905+00:00",
  "reader": "Site owners and analysts investigating a robots-related report labels a request as blocked or compliant without retaining the applicable directive and the actual response.",
  "related": [
    "robots-file-errors-vs-page-errors",
    "waf-ai-crawl-control-rule-order"
  ],
  "related_links": [
    {
      "path": "/ai-analytics/guides/robots-file-errors-vs-page-errors/",
      "title": "Separate robots.txt failures from page errors"
    }
  ],
  "research_document_ids": [
    "853226c79d02f778087874bd61631c51"
  ],
  "slug": "robots-directives-vs-access-enforcement",
  "source_signature": "3ceeeb3415f763b4b0bfc27e59ed3075d0165061fa6d45fca7f8647571219fef",
  "sources": [
    {
      "hash": "422a2f231f5d178a35ca6fd5552b7ae7fa905952d605c92634e2d68e1ae289ff",
      "locator": "Track robots.txt violations / b22; Track robots.txt violations / b25; Track robots.txt violations / b26; Track robots.txt violations / b29",
      "observed_at": "2026-11-02T17:31:04.468187+00:00",
      "title": "Directives",
      "url": "https://developers.cloudflare.com/ai-crawl-control/features/track-robots-txt/"
    }
  ],
  "sources_required": [],
  "steps": [
    "Save the robots.txt version applicable to the interval, including hostname and retrieval date. Keep later policy edits as separate versions.",
    "Identify the requested path and the exact directive used for the report classification. Preserve uncertainty when the historical file or crawler identity cannot be established.",
    "Check the response and any enforcing security rule separately. Label rule violation, attempted access, blocked response and delivered content only when each has support.",
    "If access enforcement is required, implement a scoped control under the site owner’s policy and verify it with allowed and disallowed test cases. Recheck the published directive as its own step."
  ],
  "title": "Separate robots directives from observed access enforcement",
  "worked_example": "Hypothetical example: an access record shows a request to a disallowed path. The report can preserve the applicable rule and investigate the request, but it cannot call the content delivered without a response record. Similarly, a blocked HTTP response does not establish that a crawler read or honored robots.txt. The investigation needs the relevant version of the rule, not a later edited file substituted into history."
}