from tooltrace_verifier import ToolTraceVerifier

detector = ToolTraceVerifier.from_pretrained("Resonans/tooltrace-verifier", load_lettuce=True)

result = detector.predict(
    query="Help me check the weather in Beijing.",
    context='Weather_API: {"location": "Beijing", "weather": "sunny"}',
    output="The weather in Beijing is rainy.",
    available_tools="Weather_API: checks weather by location",
    hallucination_type="contradiction",
)

print(result["spans"])

LookBackLens component

This repo includes the trained LookBackLens sklearn classifiers in lookback_lens/. The large base LM is not included:

meta-llama/Llama-3.1-8B-Instruct

Full LookBackLens attention-feature extraction/evaluation is reproduced in the project notebooks.

Routes

json

{
  "contradiction": {
    "threshold": 0.52,
    "selected_candidate": "sum__lettuce_contradiction:1__lookback_lens__tool_output:1__tool_rules:0",
    "candidate_meta": {
      "type": "weighted_sum",
      "components": [
        "lettuce_contradiction",
        "lookback_lens__tool_output",
        "tool_rules"
      ],
      "weights": [
        1.0,
        1.0,
        0.0
      ]
    },
    "validation_metrics": {
      "dataset": "contradiction",
      "method": "tooltrace_verifier",
      "context_mode": "hybrid",
      "split": "lettuce_group_validation",
      "n_examples": 1571,
      "n_gold_hallucinated": 787,
      "n_pred_hallucinated": 748,
      "example_precision": 0.9799465240641712,
      "example_recall": 0.9313850063532402,
      "example_f1": 0.9550488599348534,
      "example_accuracy": 0.9560789306174411,
      "span_precision": 0.7990827045420921,
      "span_recall": 0.8792121113462478,
      "span_f1": 0.8372345372810418,
      "span_macro_precision": 0.8643177693267186,
      "span_macro_recall": 0.9352901268265252,
      "span_macro_f1": 0.8853980289420389,
      "span_total_tp_chars": 5401,
      "span_total_pred_chars": 6759,
      "span_total_gold_chars": 6143,
      "threshold": 0.52
    },
    "test_metrics": {
      "dataset": "contradiction",
      "method": "tooltrace_verifier",
      "context_mode": "hybrid",
      "split": "lettuce_group_test",
      "n_examples": 2747,
      "n_gold_hallucinated": 1372,
      "n_pred_hallucinated": 1312,
      "example_precision": 0.9763719512195121,
      "example_recall": 0.9336734693877551,
      "example_f1": 0.9545454545454546,
      "example_accuracy": 0.9555879140880961,
      "span_precision": 0.7633746567934104,
      "span_recall": 0.8405093440820813,
      "span_f1": 0.8000872029649008,
      "span_macro_precision": 0.8618285944573578,
      "span_macro_recall": 0.9301384040407126,
      "span_macro_f1": 0.8802671907055231,
      "span_total_tp_chars": 9175,
      "span_total_pred_chars": 12019,
      "span_total_gold_chars": 10916,
      "threshold": 0.52
    },
    "lettuce_checkpoint": "lettuce/contradiction",
    "lookback_context_mode": "tool_output",
    "lookback_classifier": "lookback_lens/dataset_contradiction_raw__ctx-tool_output__src-context__w4s4__max2048__lettuce_group_split__solver-lbfgs__iter1000__tol0.0001__C1.joblib",
    "rules_enabled": true
  },
  "missing_tool": {
    "threshold": 0.36000000000000004,
    "selected_candidate": "sum__lettuce_missing_tool:0.75__lookback_lens__tools_plus_tool_output:1__tool_rules:1",
    "candidate_meta": {
      "type": "weighted_sum",
      "components": [
        "lettuce_missing_tool",
        "lookback_lens__tools_plus_tool_output",
        "tool_rules"
      ],
      "weights": [
        0.75,
        1.0,
        1.0
      ]
    },
    "validation_metrics": {
      "dataset": "missing_tool",
      "method": "tooltrace_verifier",
      "context_mode": "hybrid",
      "split": "lettuce_group_validation",
      "n_examples": 1555,
      "n_gold_hallucinated": 778,
      "n_pred_hallucinated": 798,
      "example_precision": 0.968671679197995,
      "example_recall": 0.993573264781491,
      "example_f1": 0.9809644670050761,
      "example_accuracy": 0.9807073954983923,
      "span_precision": 0.8428213309024613,
      "span_recall": 0.9438050274339671,
      "span_f1": 0.8904592788779871,
      "span_macro_precision": 0.914277505332574,
      "span_macro_recall": 0.9588100149090778,
      "span_macro_f1": 0.9281479903401246,
      "span_total_tp_chars": 36983,
      "span_total_pred_chars": 43880,
      "span_total_gold_chars": 39185,
      "threshold": 0.36000000000000004
    },
    "test_metrics": {
      "dataset": "missing_tool",
      "method": "tooltrace_verifier",
      "context_mode": "hybrid",
      "split": "lettuce_group_test",
      "n_examples": 2733,
      "n_gold_hallucinated": 1367,
      "n_pred_hallucinated": 1397,
      "example_precision": 0.9742304939155333,
      "example_recall": 0.9956108266276518,
      "example_f1": 0.9848046309696092,
      "example_accuracy": 0.9846322722283205,
      "span_precision": 0.8427726340807292,
      "span_recall": 0.9427240059598586,
      "span_f1": 0.8899507015547972,
      "span_macro_precision": 0.9164472653398278,
      "span_macro_recall": 0.9608651994218956,
      "span_macro_f1": 0.9305490153879314,
      "span_total_tp_chars": 64537,
      "span_total_pred_chars": 76577,
      "span_total_gold_chars": 68458,
      "threshold": 0.36000000000000004
    },
    "lettuce_checkpoint": "lettuce/missing_tool",
    "lookback_context_mode": "tools_plus_tool_output",
    "lookback_classifier": "lookback_lens/dataset_missing_tool_raw__ctx-tools_plus_tool_output__src-context__w8s8__max2048__lettuce_group_split.joblib",
    "rules_enabled": true
  },
  "overgeneration": {
    "threshold": 0.52,
    "selected_candidate": "sum__lettuce_overgeneration:1__lookback_lens__tool_output:1__tool_rules:0",
    "candidate_meta": {
      "type": "weighted_sum",
      "components": [
        "lettuce_overgeneration",
        "lookback_lens__tool_output",
        "tool_rules"
      ],
      "weights": [
        1.0,
        1.0,
        0.0
      ]
    },
    "validation_metrics": {
      "dataset": "overgeneration",
      "method": "tooltrace_verifier",
      "context_mode": "hybrid",
      "split": "lettuce_group_validation",
      "n_examples": 1550,
      "n_gold_hallucinated": 779,
      "n_pred_hallucinated": 784,
      "example_precision": 0.9770408163265306,
      "example_recall": 0.9833119383825417,
      "example_f1": 0.9801663467690339,
      "example_accuracy": 0.98,
      "span_precision": 0.8689510433062921,
      "span_recall": 0.9347863841611671,
      "span_f1": 0.900667238386078,
      "span_macro_precision": 0.9229057843113527,
      "span_macro_recall": 0.962588027284481,
      "span_macro_f1": 0.932678173966709,
      "span_total_tp_chars": 21530,
      "span_total_pred_chars": 24777,
      "span_total_gold_chars": 23032,
      "threshold": 0.52
    },
    "test_metrics": {
      "dataset": "overgeneration",
      "method": "tooltrace_verifier",
      "context_mode": "hybrid",
      "split": "lettuce_group_test",
      "n_examples": 2716,
      "n_gold_hallucinated": 1356,
      "n_pred_hallucinated": 1353,
      "example_precision": 0.9852180339985218,
      "example_recall": 0.9830383480825958,
      "example_f1": 0.9841269841269841,
      "example_accuracy": 0.9841678939617083,
      "span_precision": 0.8816129107271585,
      "span_recall": 0.9298310156230839,
      "span_f1": 0.9050802139037434,
      "span_macro_precision": 0.9300223131199776,
      "span_macro_recall": 0.9660561378593199,
      "span_macro_f1": 0.9381312056411896,
      "span_total_tp_chars": 37912,
      "span_total_pred_chars": 43003,
      "span_total_gold_chars": 40773,
      "threshold": 0.52
    },
    "lettuce_checkpoint": "lettuce/overgeneration",
    "lookback_context_mode": "tool_output",
    "lookback_classifier": "lookback_lens/dataset_overgeneration_raw__ctx-tool_output__src-context__w4s4__max2048__lettuce_group_split__solver-lbfgs__iter1000__tol0.0001__C1.joblib",
    "rules_enabled": true
  }
}

Results

Could not render results table: ImportError: Missing optional dependency 'tabulate'. Use pip or conda to install tabulate.

Repository contents

text

lettuce/              Fine-tuned LettuceDetect checkpoints
lookback_lens/        sklearn/joblib LookBackLens classifiers
rules/                Symbolic tool-aware rule config
tooltrace_verifier/   Inference code
results/              Evaluation tables and predictions
config.json           Composite model config
hybrid_config.json    Validation-tuned hybrid route config
thresholds.json       Per-route thresholds

Notes

This is not a single Transformer checkpoint. It is a composite detector published as one Hugging Face model repository for reproducibility and convenience.

Thresholds and routing decisions were selected on the validation split. Test results are reported only after final route selection.