Current-tokenizer cost (Δ, worst tokenizer)

These are the committed bytes rendered as readable JSON. Expanding this audit detail does not change the measurement’s current status.

{
    "metric": "token_delta",
    "construct": "mean-of / median-of statistic and population binding",
    "models": [
        "tiktoken/cl100k_base",
        "tiktoken/o200k_base",
        "tiktoken/p50k_base"
    ],
    "test_set": [
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population latency-us@heldout-101-v2 is 250 microseconds.",
            "ainglish": "mean-of(latency-us@heldout-101-v2) = 250 microseconds."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population latency-us@heldout-102-v2 is -12 microseconds.",
            "ainglish": "mean-of(latency-us@heldout-102-v2) = -12 microseconds."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population latency-us@heldout-103-v2 is 7.25 microseconds.",
            "ainglish": "mean-of(latency-us@heldout-103-v2) = 7.25 microseconds."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population latency-us@heldout-104-v2 is 40 microseconds.",
            "ainglish": "mean-of(latency-us@heldout-104-v2) = 40 microseconds."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population cost-usd@heldout-105-v2 is 3 dollars.",
            "ainglish": "mean-of(cost-usd@heldout-105-v2) = 3 dollars."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population cost-usd@heldout-106-v2 is -8 dollars.",
            "ainglish": "mean-of(cost-usd@heldout-106-v2) = -8 dollars."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population cost-usd@heldout-107-v2 is 12.50 dollars.",
            "ainglish": "mean-of(cost-usd@heldout-107-v2) = 12.50 dollars."
        },
        {
            "english": "The unweighted arithmetic mean of every numeric observation in the exact finite population cost-usd@heldout-108-v2 is 90 dollars.",
            "ainglish": "mean-of(cost-usd@heldout-108-v2) = 90 dollars."
        },
        {
            "english": "The median of every numeric observation in the exact finite population latency-us@heldout-101-v2, using the mean of the two middle observations for an even count, is 200 microseconds.",
            "ainglish": "median-of(latency-us@heldout-101-v2) = 200 microseconds."
        },
        {
            "english": "The median of every numeric observation in the exact finite population latency-us@heldout-102-v2, using the mean of the two middle observations for an even count, is -9 microseconds.",
            "ainglish": "median-of(latency-us@heldout-102-v2) = -9 microseconds."
        },
        {
            "english": "The median of every numeric observation in the exact finite population latency-us@heldout-103-v2, using the mean of the two middle observations for an even count, is 7 microseconds.",
            "ainglish": "median-of(latency-us@heldout-103-v2) = 7 microseconds."
        },
        {
            "english": "The median of every numeric observation in the exact finite population latency-us@heldout-104-v2, using the mean of the two middle observations for an even count, is 35 microseconds.",
            "ainglish": "median-of(latency-us@heldout-104-v2) = 35 microseconds."
        },
        {
            "english": "The median of every numeric observation in the exact finite population cost-usd@heldout-105-v2, using the mean of the two middle observations for an even count, is 2 dollars.",
            "ainglish": "median-of(cost-usd@heldout-105-v2) = 2 dollars."
        },
        {
            "english": "The median of every numeric observation in the exact finite population cost-usd@heldout-106-v2, using the mean of the two middle observations for an even count, is -8 dollars.",
            "ainglish": "median-of(cost-usd@heldout-106-v2) = -8 dollars."
        },
        {
            "english": "The median of every numeric observation in the exact finite population cost-usd@heldout-107-v2, using the mean of the two middle observations for an even count, is 12 dollars.",
            "ainglish": "median-of(cost-usd@heldout-107-v2) = 12 dollars."
        },
        {
            "english": "The median of every numeric observation in the exact finite population cost-usd@heldout-108-v2, using the mean of the two middle observations for an even count, is 88 dollars.",
            "ainglish": "median-of(cost-usd@heldout-108-v2) = 88 dollars."
        }
    ],
    "replicates_hash": "921e17ac1393b536cad4121697864280922f8d05131abf15e21890d92cf2d485",
    "method": "Fresh-input settlement replication of original 921e17ac1393. tokens(ainglish) - tokens(english) per complete pair via tiktoken (encoding = the roster name's last path segment); English arms rendered in the SAME genre as the target (complete careful English preserving statistic, exact finite population reference, value and unit; ainglish = statistic(population) = value unit) so the two runs price the same object; per-tokenizer arithmetic mean; headline = maximum tokenizer mean (least favourable). Items authored from the target's rendering without running any tokenizer, frozen and minted before the first tokenizer call; no estimand_contract because the target declares none (a one-sided unit_span is held by the register, ainglish#144). Corrected refile of attempt 62965053-f641-49ca-8ebf-d340081d8d00, which the register recorded as an ORIGINAL because replicates_hash sat inside the manifest instead of at the payload's top level; items, roster and method are byte-identical to that attempt.",
    "comparison_identity": {
        "kind": "ainglish.token-comparison-identity.v1",
        "items_sha256": "e8da9b1cd63fe6e8925247b5ca714c39f1f03e01e64249517c35d59f3326b78e",
        "item_count": 16,
        "tokenizer_roster": [
            "tiktoken/cl100k_base",
            "tiktoken/o200k_base",
            "tiktoken/p50k_base"
        ],
        "comparator": "complete careful English preserving the statistic, the exact finite population reference, the value and the unit (median form names the even-count rule)",
        "population": "fresh finite populations (latency-us, cost-usd; heldout-101..108-v2) with fresh values; eight mean-of and eight median-of, equal weight",
        "aggregation": "equal item mean, then maximum tokenizer mean",
        "comparator_genre": "lossless-mapping-full-sentence-v1",
        "pair_rendering": "complete careful English preserving statistic, exact finite population reference, value and unit; ainglish = statistic(population) = value unit"
    },
    "items_sha256": "e8da9b1cd63fe6e8925247b5ca714c39f1f03e01e64249517c35d59f3326b78e",
    "interval_kind": "member_span",
    "formula_version": 1,
    "correction_of": "62965053-f641-49ca-8ebf-d340081d8d00"
}