{"service":"wever-agent-evaluation-checks-v1","category":"Agent Evaluation","family":"Agent Evaluation","page":"https://weverlabs.com/agent-evaluation/","tools":{"wever_evaluation-design-check":{"path":"/api/evaluation-design-check","kind":"check","receipt_type":"wever.evaluation-design-check.v1","schema":{"type":"object","additionalProperties":false,"required":["criteria"],"properties":{"plan":{"type":"object","additionalProperties":false,"required":[],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The evaluation."},"agent":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The agent or service evaluated."},"task":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The task, such as answering billing questions."},"written_at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When the criteria and thresholds were written."},"run_at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When the evaluation first ran."}}},"tasks":{"type":"object","additionalProperties":false,"required":[],"properties":{"size":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks in the set."},"source":{"enum":["real_usage","expert_written","synthetic","public_benchmark","mixed"],"description":"Real usage, expert-written, generated, a public benchmark or a mix."},"described":{"type":"boolean","description":"Where the tasks come from and what they cover is written down."}}},"criteria":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"object","additionalProperties":false,"required":["id","text"],"properties":{"id":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"text":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The criterion as written."},"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The metric that measures it, by name."}}},"description":"Each success criterion."},"metrics":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The metric, such as resolution rate."},"measures":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"What it measures."},"computed":{"type":"boolean","description":"How it is computed is written down."},"threshold":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"The pass threshold, such as 85."},"direction":{"enum":["at_least","at_most"],"description":"A pass is at least or at most the threshold."},"threshold_source":{"enum":["requirement","baseline","prior_run","benchmark","none"],"description":"A requirement, the baseline’s score, a prior run, a benchmark, or none."}}},"description":"Each metric."},"baseline":{"type":"object","additionalProperties":false,"required":[],"properties":{"kind":{"enum":["prior_version","human","simple_method","other_system","none"],"description":"The prior version, people, a simple method, another system, or none."},"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The baseline."},"scores":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"object","additionalProperties":false,"required":["metric","value"],"properties":{"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000}}},"description":"The baseline’s score per metric."}}},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["plan","tasks","criteria","metrics","baseline","findings","fixes"],"verdicts":["untestable_plan","design_gaps","design_unverified","design_complete"]},"wever_evaluation-dataset-check":{"path":"/api/evaluation-dataset-check","kind":"check","receipt_type":"wever.evaluation-dataset-check.v1","schema":{"type":"object","additionalProperties":false,"required":["dataset"],"properties":{"dataset":{"type":"object","additionalProperties":false,"required":[],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The evaluation set."},"size":{"type":"integer","minimum":0,"maximum":100000000,"description":"Examples in the set."},"source":{"enum":["production_logs","expert_written","synthetic","public_benchmark","mixed"],"description":"Production logs, expert-written, generated, a public benchmark or a mix."},"version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The dataset version."},"published_at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When a public benchmark was first published."}}},"coverage":{"type":"object","additionalProperties":false,"required":[],"properties":{"required":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"description":"The categories the agent must handle."},"covered":{"type":"array","minItems":0,"maxItems":200,"items":{"type":"object","additionalProperties":false,"required":["category","examples"],"properties":{"category":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"examples":{"type":"integer","minimum":0,"maximum":100000000}}},"description":"Each category with its example count."},"hard_cases":{"type":"boolean","description":"Hard and edge cases are included on purpose."}}},"labeling":{"type":"object","additionalProperties":false,"required":[],"properties":{"method":{"enum":["expert","trained_annotators","crowd","model","author","none"],"description":"Experts, trained annotators, crowd workers, a model, the agent’s builders, or no one."},"labelers":{"type":"integer","minimum":0,"maximum":100000000,"description":"Labelers per example."},"agreement":{"type":"number","minimum":0,"maximum":1,"description":"Labeler agreement, such as 0.72."},"guideline":{"type":"boolean","description":"A written labeling guide is used."},"reviewed":{"type":"boolean","description":"A person reviewed model-made labels."}}},"leakage":{"type":"object","additionalProperties":false,"required":[],"properties":{"split":{"enum":["held_out","same_as_training","not_split"],"description":"Held out from training and tuning, the same data, or never split."},"overlap_checked":{"type":"boolean","description":"Overlap with training, tuning and prompt examples was checked."},"overlap_found":{"type":"integer","minimum":0,"maximum":100000000,"description":"Examples also in training or tuning data."},"prompt_examples_from_set":{"type":"boolean","description":"Prompt examples come from this set."},"tuned_on_set":{"type":"boolean","description":"Anything was tuned against this set."},"model_cutoff":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The model’s training data cutoff."}}},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["dataset","coverage","labeling","leakage","findings","fixes"],"verdicts":["dataset_unfit","dataset_gaps","dataset_unverified","dataset_sound"]},"wever_evaluation-scoring-check":{"path":"/api/evaluation-scoring-check","kind":"check","receipt_type":"wever.evaluation-scoring-check.v1","schema":{"type":"object","additionalProperties":false,"required":["rubric"],"properties":{"rubric":{"type":"object","additionalProperties":false,"required":[],"properties":{"version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The rubric version."},"criteria":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"description":"What is scored."},"scale":{"type":"object","additionalProperties":false,"required":["min","max"],"properties":{"min":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"Lowest score."},"max":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"Highest score."},"step":{"type":"number","minimum":0.001,"maximum":1000000,"description":"Step between points; 1 when left out."}}},"points":{"type":"array","minItems":0,"maxItems":21,"items":{"type":"object","additionalProperties":false,"required":["score"],"properties":{"score":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"The scale point."},"anchor":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"What a response at this point looks like."},"example":{"type":"boolean","description":"A worked example is attached."}}},"description":"Each scale point."}}},"judge":{"type":"object","additionalProperties":false,"required":[],"properties":{"kind":{"enum":["model","human","automatic","mixed"],"description":"A model, people, an automatic metric, or a mix."},"model":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The judge model."},"version_pinned":{"type":"boolean","description":"The judge version is pinned."},"prompt_fixed":{"type":"boolean","description":"The judge prompt is fixed and versioned."},"temperature":{"type":"number","minimum":0,"maximum":10,"description":"The judge’s temperature."},"same_as_agent":{"type":"boolean","description":"The judge is the model being scored."},"calibration":{"type":"object","additionalProperties":false,"required":[],"properties":{"against":{"enum":["human_labels","gold_answers","none"],"description":"Human labels, known answers, or nothing."},"items":{"type":"integer","minimum":0,"maximum":100000000,"description":"Calibration items."},"agreement":{"type":"number","minimum":0,"maximum":1,"description":"Agreement with them, such as 0.84."}}}}},"spot_check":{"type":"object","additionalProperties":false,"required":[],"properties":{"done":{"type":"boolean","description":"A person checks a share of the scores."},"share":{"type":"number","minimum":0,"maximum":1,"description":"Share checked, such as 0.1."},"by":{"enum":["independent","author"],"description":"Someone independent of the builders, or the builders."},"disagreement":{"type":"number","minimum":0,"maximum":1,"description":"Share of checked scores disputed."}}},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["rubric","judge","spot_check","comparability","findings","fixes"],"verdicts":["scores_unreliable","scoring_gaps","scoring_unverified","scoring_consistent"]},"wever_evaluation-result-check":{"path":"/api/evaluation-result-check","kind":"check","receipt_type":"wever.evaluation-result-check.v1","schema":{"type":"object","additionalProperties":false,"required":["claims"],"properties":{"claims":{"type":"array","minItems":1,"maxItems":200,"items":{"type":"object","additionalProperties":false,"required":["id","metric","value"],"properties":{"id":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"text":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The claim as written."},"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The metric."},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"The figure claimed."},"subset":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The subset the figure comes from, if not the full set."},"subset_disclosed":{"type":"boolean","description":"The claim names the subset."},"compared_to":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The baseline compared with."},"direction":{"enum":["higher","lower","meets_threshold"],"description":"Higher or lower than the baseline, or meets a threshold."},"threshold":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"The threshold claimed met."}}},"description":"Each claim."},"results":{"type":"array","minItems":0,"maxItems":500,"items":{"type":"object","additionalProperties":false,"required":["metric","value"],"properties":{"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000,"description":"The measured score."},"subset":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The subset, if not the full set."},"n":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks scored."},"spread":{"type":"object","additionalProperties":false,"required":["kind","value"],"properties":{"kind":{"enum":["std","stderr","ci95"],"description":"A standard deviation, a standard error, or a 95 percent interval half-width."},"value":{"type":"number","minimum":0,"maximum":1000000000,"description":"In the metric’s units, such as 2.1."}}},"conditions":{"type":"object","additionalProperties":false,"required":[],"properties":{"model":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Model and version."},"dataset":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Dataset and version."},"run_at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When it ran."}}}}},"description":"Each measured result."},"baselines":{"type":"array","minItems":0,"maxItems":500,"items":{"type":"object","additionalProperties":false,"required":["name","metric","value"],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000},"n":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks scored."},"spread":{"type":"object","additionalProperties":false,"required":["kind","value"],"properties":{"kind":{"enum":["std","stderr","ci95"],"description":"A standard deviation, a standard error, or a 95 percent interval half-width."},"value":{"type":"number","minimum":0,"maximum":1000000000,"description":"In the metric’s units, such as 2.1."}}}}},"description":"Each baseline result."},"subsets":{"type":"object","additionalProperties":false,"required":[],"properties":{"run":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"description":"Every subset that was run."},"reported":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"description":"The subsets the report shows."}}},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["claims","results","subsets","findings","fixes"],"verdicts":["claims_unsupported","result_gaps","result_unverified","claims_supported"]},"wever_evaluation-reproducibility-check":{"path":"/api/evaluation-reproducibility-check","kind":"check","receipt_type":"wever.evaluation-reproducibility-check.v1","schema":{"type":"object","additionalProperties":false,"required":["agent"],"properties":{"agent":{"type":"object","additionalProperties":false,"required":[],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The agent evaluated."},"model":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The model."},"model_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The exact version, such as a dated snapshot."},"provider":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Who serves the model."}}},"prompts":{"type":"object","additionalProperties":false,"required":[],"properties":{"included":{"type":"boolean","description":"The full prompts, templates and examples are included."},"version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The prompt version."}}},"settings":{"type":"object","additionalProperties":false,"required":[],"properties":{"temperature":{"type":"number","minimum":0,"maximum":10},"top_p":{"type":"number","minimum":0,"maximum":1},"max_tokens":{"type":"integer","minimum":0,"maximum":100000000},"seed":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The seed, or none."},"other":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"description":"Other settings."}}},"tools":{"type":"object","additionalProperties":false,"required":[],"properties":{"listed":{"type":"boolean","description":"The tools are listed."},"versions":{"type":"boolean","description":"Tool versions are stated."},"external":{"enum":["none","recorded","mocked","live"],"description":"Outside calls: none, recorded and replayed, mocked, or live."}}},"dataset":{"type":"object","additionalProperties":false,"required":[],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"hash":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"A content hash."},"access":{"enum":["public","on_request","private"],"description":"Anyone, on request, or no one outside the team."}}},"environment":{"type":"object","additionalProperties":false,"required":[],"properties":{"code_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Such as a commit id."},"dependencies_pinned":{"type":"boolean","description":"Package versions are pinned."},"runtime":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"}}},"scoring":{"type":"object","additionalProperties":false,"required":[],"properties":{"method":{"enum":["script","model_judge","human","mixed"],"description":"A script, a model judge, people, or a mix."},"script_included":{"type":"boolean","description":"The scoring script is included."},"judge_model":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"judge_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The pinned judge version."}}},"run_at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When it ran."},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["packet","essentials","missing","findings","fixes"],"verdicts":["cannot_rerun","reproducibility_gaps","reproducibility_unverified","rerunnable"]},"wever_evaluation-drift-check":{"path":"/api/evaluation-drift-check","kind":"check","receipt_type":"wever.evaluation-drift-check.v1","schema":{"type":"object","additionalProperties":false,"required":["earlier","later"],"properties":{"agent":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The agent both runs evaluate."},"earlier":{"type":"object","additionalProperties":false,"required":["id","scores"],"properties":{"id":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When the run happened."},"model":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"model_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"prompt_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"dataset":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"dataset_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"judge":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"judge_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"rubric_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"temperature":{"type":"number","minimum":0,"maximum":10},"scores":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"object","additionalProperties":false,"required":["metric","value"],"properties":{"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000},"n":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks scored."},"spread":{"type":"object","additionalProperties":false,"required":["kind","value"],"properties":{"kind":{"enum":["std","stderr","ci95"],"description":"A standard deviation, a standard error, or a 95 percent interval half-width."},"value":{"type":"number","minimum":0,"maximum":1000000000,"description":"In the metric’s units, such as 2.1."}}}}},"description":"Each score."}},"description":"The earlier run."},"later":{"type":"object","additionalProperties":false,"required":["id","scores"],"properties":{"id":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"at":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"When the run happened."},"model":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"model_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"prompt_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"dataset":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"dataset_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"judge":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"judge_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"rubric_version":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"temperature":{"type":"number","minimum":0,"maximum":10},"scores":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"object","additionalProperties":false,"required":["metric","value"],"properties":{"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000},"n":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks scored."},"spread":{"type":"object","additionalProperties":false,"required":["kind","value"],"properties":{"kind":{"enum":["std","stderr","ci95"],"description":"A standard deviation, a standard error, or a 95 percent interval half-width."},"value":{"type":"number","minimum":0,"maximum":1000000000,"description":"In the metric’s units, such as 2.1."}}}}},"description":"Each score."}},"description":"The later run."},"tolerance":{"type":"number","minimum":0,"maximum":1000000000,"description":"How far a score may move before it counts as drift, such as 2."},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["runs","conditions","scores","findings","fixes"],"verdicts":["runs_not_comparable","scores_drifted","drift_unverified","scores_stable"]},"wever_evaluation-cost-check":{"path":"/api/evaluation-cost-check","kind":"check","receipt_type":"wever.evaluation-cost-check.v1","schema":{"type":"object","additionalProperties":false,"required":["budget"],"properties":{"budget":{"type":"object","additionalProperties":false,"required":[],"properties":{"total":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The run budget, such as 500 USDC."},"per_task":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The cost cap per task."},"latency_per_task":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"The time cap per task, such as 30 seconds."},"latency_basis":{"enum":["p50","p95","max"],"description":"The time cap applies to the median, the 95th percentile or the slowest task."}}},"run":{"type":"object","additionalProperties":false,"required":[],"properties":{"tasks":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks scored."},"total_cost":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Such as 612 USDC."},"latency":{"type":"object","additionalProperties":false,"required":[],"properties":{"p50":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"p95":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"max":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"}}}}},"components":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Such as judge model calls."},"kind":{"enum":["agent_model","judge_model","tools","compute","human_review","storage","other"]},"cost":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Such as 140 USDC."}}},"description":"Each cost component."},"tasks":{"type":"array","minItems":0,"maxItems":1000,"items":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"cost":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"latency":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"}}},"description":"Per-task figures."},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["budget","run","components","tasks","findings","fixes"],"verdicts":["over_budget","cost_gaps","cost_unverified","within_budget"]},"wever_evaluation-report-check":{"path":"/api/evaluation-report-check","kind":"check","receipt_type":"wever.evaluation-report-check.v1","schema":{"type":"object","additionalProperties":false,"required":["headline"],"properties":{"report":{"type":"object","additionalProperties":false,"required":[],"properties":{"title":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"audience":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S","description":"Who it is written for."}}},"headline":{"type":"array","minItems":1,"maxItems":50,"items":{"type":"object","additionalProperties":false,"required":["metric","value"],"properties":{"metric":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"value":{"type":"number","minimum":-1000000000,"maximum":1000000000},"spread_shown":{"type":"boolean","description":"Its spread is shown beside it."},"sample_size_shown":{"type":"boolean","description":"Its sample size is shown beside it."},"baseline_shown":{"type":"boolean","description":"A baseline is shown beside it."}}},"description":"Each headline score."},"limits":{"type":"object","additionalProperties":false,"required":[],"properties":{"disclosed":{"type":"boolean","description":"What the evaluation does not cover is stated."},"placement":{"enum":["alongside","later_section","appendix","separate_document"],"description":"Beside the headline, a later section, an appendix or a separate document."},"items":{"type":"array","minItems":0,"maxItems":100,"items":{"type":"string","minLength":1,"maxLength":500,"pattern":"\\S"},"description":"Each limit stated."}}},"failures":{"type":"object","additionalProperties":false,"required":[],"properties":{"disclosed":{"type":"boolean","description":"Failures are reported."},"known":{"type":"integer","minimum":0,"maximum":100000000,"description":"Tasks that failed in the run."},"reported":{"type":"integer","minimum":0,"maximum":100000000,"description":"Failures the summary reports."},"examples":{"type":"boolean","description":"Failure examples are shown."},"placement":{"enum":["alongside","later_section","appendix","separate_document"]}}},"conditions":{"type":"object","additionalProperties":false,"required":[],"properties":{"model":{"type":"boolean","description":"Model and version stated."},"dataset":{"type":"boolean","description":"Dataset and version stated."},"run_at":{"type":"boolean","description":"Run date stated."},"judge":{"type":"boolean","description":"Scoring method stated."},"settings":{"type":"boolean","description":"Settings stated."}}},"full_results":{"enum":["linked","on_request","none"],"description":"Linked, on request, or not available."},"label":{"type":"string","minLength":1,"maxLength":200,"pattern":"\\S","description":"Optional name recorded in the report."}}},"reads":"material in the call","elements":["headline","disclosure","findings","fixes"],"verdicts":["headline_only","disclosure_gaps","disclosure_unverified","fully_disclosed"]}},"free_tools":{"endpoint":"https://weverlabs.com/mcp-free","output":"unsigned; the final line names the main /mcp tools","tools":{"wever_evaluation-snapshot":{"reads":"an agent or service name and a task type","returns":"the checklist of what a complete evaluation packet for that task holds, part by part, with the family check that reads each part"},"wever_evaluation-rubric-sample":{"samples":"wever_evaluation-scoring-check","reads":"an optional sample task","returns":"one worked scoring rubric with anchored scale points and a scored example, with its anchoring read by the scoring check"}}},"limits":{"all_supplied_text":"512 KB per call","report":"1 MB","max_request_bytes":3211264,"names":100,"criteria":100,"metrics":100,"categories":200,"points":21,"claims":200,"results":500,"scores":100,"components":100,"tasks":1000,"headline":50},"material":"Each tool reads only the material in the call: an evaluation plan with its task set, criteria, metrics, thresholds and baseline, an evaluation dataset with its coverage, labeling and leakage controls, a scoring rubric with its judge setup and spot checks, a result report with its claims and measured figures, a reproducibility packet, two runs of the same agent, a run’s cost and latency figures against its budget, or a summary written for a buyer or principal. Every status, match and verdict is decided from that material; the report makes no network request or fetch, runs no evaluation, calls no model or judge, and moves no money, writes nothing outside the report, contacts no agent or counterparty and changes no configuration.","basis":"This report reads only the evaluation plans, datasets, rubrics, judge setups, results, reproducibility packets, runs, cost figures and summaries in the material in the call, and decides every status, match and verdict from them.","related":{"deliverable":"wever_deliverable-check","spec_drift":"wever_spec-drift-check","benchmark_evidence":"wever_benchmark-evidence-check","claim_evidence":"wever_claim-evidence-check","result_validation":"wever_result-validation-check","trust_scorecard":"wever_agent-trust-scorecard","endpoint_reliability":"wever_endpoint-reliability-check","budget_controls":"wever_budget-controls-check","spending_anomaly":"wever_spending-anomaly-check","budget_forecast":"wever_budget-forecast-brief","ai_disclosure":"wever_ai-disclosure-check","recommendation_disclosure":"wever_recommendation-disclosure-check"},"pricing":{"network":"eip155:8453","asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","pricesAtomic":{"/api/evaluation-design-check":"100000","/api/evaluation-dataset-check":"100000","/api/evaluation-scoring-check":"100000","/api/evaluation-result-check":"100000","/api/evaluation-reproducibility-check":"100000","/api/evaluation-drift-check":"100000","/api/evaluation-cost-check":"100000","/api/evaluation-report-check":"100000"},"x402Version":2,"scheme":"exact","freeTier":{"calls":10,"windowDays":30,"scope":"all_paid_endpoints","walletProofMaxAgeSeconds":300}},"wallet_proof":"Sign the exact JSON arguments and the tool route with X-Wever-Wallet-Proof. Retry an unchanged payment or wallet-proof request to recover its stored report.","signing":{"issuer":"https://weverlabs.com/api/agent-evaluation-contract","key_id":"wever-trust-20261001","public_jwk":{"crv":"Ed25519","x":"PG6bYTJfqh-Y7YEMzxNVhMsFkrO1XnX7djPUvNi9g_U","kty":"OKP"},"algorithm":"Ed25519","receipt_types":{"evaluation-design-check":"wever.evaluation-design-check.v1","evaluation-dataset-check":"wever.evaluation-dataset-check.v1","evaluation-scoring-check":"wever.evaluation-scoring-check.v1","evaluation-result-check":"wever.evaluation-result-check.v1","evaluation-reproducibility-check":"wever.evaluation-reproducibility-check.v1","evaluation-drift-check":"wever.evaluation-drift-check.v1","evaluation-cost-check":"wever.evaluation-cost-check.v1","evaluation-report-check":"wever.evaluation-report-check.v1"},"jwt_type":"wever-agent-evaluation+jwt","digest":"SHA-256 of UTF-8 JSON.stringify(report), preserving returned property order. Verify the JWS signature and the report_sha256 and subject_sha256 claims."}}