Current-tokenizer cost (Δ, worst tokenizer)

These are the committed bytes rendered as readable JSON. Expanding this audit detail does not change the measurement’s current status.

{
    "metric": "token_delta",
    "construct": "part-chosen / part-capped",
    "models": [
        "cl100k_base",
        "o200k_base"
    ],
    "test_set": [
        {
            "stratum": "part-chosen",
            "english": "I examined the 37 invoices that the amount-band rule chose, out of the 284 issued; the rule picked which invoices to examine.",
            "ainglish": "part-chosen(amount-band): the 37 invoices."
        },
        {
            "stratum": "part-chosen",
            "english": "I inspected the 52 containers that the owner-team rule chose, out of the 419 running; the rule picked which containers to inspect.",
            "ainglish": "part-chosen(owner-team): the 52 containers."
        },
        {
            "stratum": "part-chosen",
            "english": "I reviewed the 64 commits that the sha-suffix-7 rule chose, out of the 903 pushed; the rule picked which commits to review.",
            "ainglish": "part-chosen(sha-suffix-7): the 64 commits."
        },
        {
            "stratum": "part-chosen",
            "english": "I traced the 29 shipments that the even-id rule chose, out of the 158 dispatched; the rule picked which shipments to trace.",
            "ainglish": "part-chosen(even-id): the 29 shipments."
        },
        {
            "stratum": "part-chosen",
            "english": "I interviewed the 45 applicants that the top-decile-score rule chose, out of the 512 who applied; the rule picked which applicants to interview.",
            "ainglish": "part-chosen(top-decile-score): the 45 applicants."
        },
        {
            "stratum": "part-chosen",
            "english": "I calibrated the 88 sensors that the first-response rule chose, out of the 640 installed; the rule picked which sensors to calibrate.",
            "ainglish": "part-chosen(first-response): the 88 sensors."
        },
        {
            "stratum": "part-chosen",
            "english": "I probed the 33 domains that the vendor-list rule chose, out of the 271 registered; the rule picked which domains to probe.",
            "ainglish": "part-chosen(vendor-list): the 33 domains."
        },
        {
            "stratum": "part-chosen",
            "english": "I transcribed the 26 recordings that the month-of-june rule chose, out of the 190 archived; the rule picked which recordings to transcribe.",
            "ainglish": "part-chosen(month-of-june): the 26 recordings."
        },
        {
            "stratum": "part-capped",
            "english": "I scanned 250 of the 734 repositories; the listing stopped at 250 per page, so I could not scan the remaining 484 repositories.",
            "ainglish": "part-capped(page-size-250): the 250 repositories."
        },
        {
            "stratum": "part-capped",
            "english": "I triaged 58 of the 366 tickets; the rate limit cut me off after sixty seconds, so I could not triage the remaining 308 tickets.",
            "ainglish": "part-capped(rate-limit-60s): the 58 tickets."
        },
        {
            "stratum": "part-capped",
            "english": "I rebuilt 41 of the 220 images; the two-gigabyte disk quota stopped the build, so I could not rebuild the remaining 179 images.",
            "ainglish": "part-capped(disk-quota-2gb): the 41 images."
        },
        {
            "stratum": "part-capped",
            "english": "I summarised 73 of the 515 messages; the eight-thousand-token budget ran out, so I could not summarise the remaining 442 messages.",
            "ainglish": "part-capped(token-budget-8k): the 73 messages."
        },
        {
            "stratum": "part-capped",
            "english": "I reconciled 96 of the 388 accounts; my read-only permission covered only the EU region, so I could not reconcile the remainder.",
            "ainglish": "part-capped(permission-read-only-eu): the 96 accounts."
        },
        {
            "stratum": "part-capped",
            "english": "I profiled 47 of the 301 nodes; the fourteen-day retention window exposed only the recent nodes, so I could not profile the older ones.",
            "ainglish": "part-capped(retention-window-14d): the 47 nodes."
        },
        {
            "stratum": "part-capped",
            "english": "I matched 100 of the 457 receipts; the interface showed at most 100 rows, so I could not match the remaining 357 receipts.",
            "ainglish": "part-capped(ui-page-cap-100): the 100 receipts."
        },
        {
            "stratum": "part-capped",
            "english": "I indexed 212 of the 690 articles; the connection dropped at row 212, so I could not index the remaining 478 articles.",
            "ainglish": "part-capped(network-cut-at-row-212): the 212 articles."
        }
    ],
    "settlement_strata": [
        {
            "id": "part-capped",
            "weight": 1
        },
        {
            "id": "part-chosen",
            "weight": 1
        }
    ],
    "estimand_contract": {
        "kind": "ainglish.estimand-shadow.v1",
        "unit_span": "complete message",
        "contrast": "Ainglish form versus complete careful English",
        "population": "16 frozen fresh pairs across part-capped and part-chosen",
        "aggregation": {
            "reducer": "least_favourable",
            "rule": "equal item mean per stratum, weighted by stratum share, then maximum tokenizer mean"
        },
        "governance_effect": "report_only"
    },
    "replicates_hash": "13a722dd4d8b0206a42ff6450c5de1fea05a0f828d14254c61889bd7af894e83",
    "comparison_identity": {
        "comparator_genre": "complete-careful-english-boundary-source-v1",
        "pair_rendering": "standalone-coverage-report",
        "kind": "ainglish.token-comparison-identity.v1",
        "items_sha256": "f6615328a57924866cdc2404198034e902ba78265faaee7dea55228a4860a5ee",
        "item_count": 16,
        "tokenizer_roster": [
            "cl100k_base",
            "o200k_base"
        ],
        "comparator": "Ainglish form versus complete careful English",
        "population": "16 frozen fresh pairs across part-capped and part-chosen",
        "aggregation": "equal item mean per stratum, weighted by stratum share, then maximum tokenizer mean",
        "unit_span": "complete message"
    },
    "method": "Fresh-input replication via the canonical ainglish-token runner (SDK 0.2.55): 16 wholly new pairs authored in the target's comparator genre (complete careful English coverage report naming the boundary source vs the Ainglish form), 8 per declared stratum; tokens(ainglish)-tokens(english) per pair under cl100k_base and o200k_base; equal item mean per stratum, weighted by stratum share, then maximum tokenizer mean.",
    "source": {
        "replicates": "13a722dd4d8b0206a42ff6450c5de1fea05a0f828d14254c61889bd7af894e83",
        "target_attempt": "8dbf38ab-f6db-4a80-848a-ecc32fb2cdab",
        "target_value": -15.5
    },
    "items_sha256": "f6615328a57924866cdc2404198034e902ba78265faaee7dea55228a4860a5ee",
    "interval_kind": "member_span",
    "tokenizer_provenance": {
        "kind": "ainglish.tiktoken-provenance.v1",
        "library": "tiktoken",
        "library_version": "0.14.0",
        "encodings": [
            "cl100k_base",
            "o200k_base"
        ]
    }
}