{ "schema_version": "1.0", "generated_at": "2026-06-22T21:01:41.003Z", "n_cells": 56, "systems": [ "mem0_v3_cloud", "mnemoverse", "mnemoverse_engine", "mnemoverse_http", "naked_cosine", "supermemory", "zep" ], "benchmarks": [ "beam", "locomo" ], "cells": [ { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k10", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k10.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5724, "judge_accuracy_by_judge": { "mnemoverse": 0.572368, "mem0": 0.684211, "mem0-4o": 0.710526, "strict": 0.256579 }, "cost_usd_total": 0.02, "wallclock_s": 1345, "git_sha": "6ee1326", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k100", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k100.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6645, "judge_accuracy_by_judge": { "mnemoverse": 0.664474, "mem0": 0.710526, "mem0-4o": 0.75, "strict": 0.276316 }, "cost_usd_total": 0.1271, "wallclock_s": 1290.73, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k20", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k20.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.625, "judge_accuracy_by_judge": { "mnemoverse": 0.625, "mem0": 0.717105, "mem0-4o": 0.736842, "strict": 0.282895 }, "cost_usd_total": 0.0317, "wallclock_s": 1199.71, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k200", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k200.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6645, "judge_accuracy_by_judge": { "mnemoverse": 0.664474, "mem0": 0.730263, "mem0-4o": 0.743421, "strict": 0.269737 }, "cost_usd_total": 0.1922, "wallclock_s": 1272.71, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv26_n152_k50", "path": "cells/cell_mem0_v3_cloud_locomo_conv26_n152_k50.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.625, "judge_accuracy_by_judge": { "mnemoverse": 0.625, "mem0": 0.703947, "mem0-4o": 0.723684, "strict": 0.269737 }, "cost_usd_total": 0.0679, "wallclock_s": 1208.91, "git_sha": "a6588c0", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_engine_beam_10m_conv0-9_n200_k200", "path": "cells/cell_mnemoverse_engine_beam_10m_conv0-9_n200_k200.json", "system": "mnemoverse_engine", "benchmark": "beam", "benchmark_scope": "10m_conv0-9", "benchmark_scope_n": 200, "role": "headline", "k": 200, "reader_model": "claude-sonnet-4-6", "primary_judge": "gpt-5", "headline_metric": 0.61, "judge_accuracy_by_judge": { "gpt-5": 0.61 }, "cost_usd_total": 1.7639, "wallclock_s": 108930.97, "git_sha": null, "eval_path": "beam_v1", "known_asymmetries": [ "BEAM-SINGLE-SEED", "BEAM-READER-MISMATCH", "BEAM-INPUT-TOKEN-ASYM" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "beam-10m-2026-05-26", "single-system", "single-seed" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": false }, { "cell_id": "cell_mnemoverse_engine_locomo_conv26_n152_k10", "path": "cells/cell_mnemoverse_engine_locomo_conv26_n152_k10.json", "system": "mnemoverse_engine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6908, "judge_accuracy_by_judge": { "mnemoverse": 0.690789, "mem0": 0.802632, "mem0-4o": 0.822368, "strict": 0.375 }, "cost_usd_total": 0.0456, "wallclock_s": 1867.49, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_engine_locomo_conv26_n152_k100", "path": "cells/cell_mnemoverse_engine_locomo_conv26_n152_k100.json", "system": "mnemoverse_engine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6645, "judge_accuracy_by_judge": { "mnemoverse": 0.664474, "mem0": 0.75, "mem0-4o": 0.802632, "strict": 0.361842 }, "cost_usd_total": 0.1426, "wallclock_s": 1822.53, "git_sha": "fb67250", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_engine_locomo_conv26_n152_k20", "path": "cells/cell_mnemoverse_engine_locomo_conv26_n152_k20.json", "system": "mnemoverse_engine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6842, "judge_accuracy_by_judge": { "mnemoverse": 0.684211, "mem0": 0.809211, "mem0-4o": 0.848684, "strict": 0.361842 }, "cost_usd_total": 0.0813, "wallclock_s": 1815.03, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_engine_locomo_conv26_n152_k200", "path": "cells/cell_mnemoverse_engine_locomo_conv26_n152_k200.json", "system": "mnemoverse_engine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6382, "judge_accuracy_by_judge": { "mnemoverse": 0.638158, "mem0": 0.743421, "mem0-4o": 0.809211, "strict": 0.375 }, "cost_usd_total": 0.1313, "wallclock_s": 1884.72, "git_sha": "5e8d805", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_engine_locomo_conv26_n152_k50", "path": "cells/cell_mnemoverse_engine_locomo_conv26_n152_k50.json", "system": "mnemoverse_engine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.7105, "judge_accuracy_by_judge": { "mnemoverse": 0.710526, "mem0": 0.769737, "mem0-4o": 0.822368, "strict": 0.388158 }, "cost_usd_total": 0.14, "wallclock_s": 1876.98, "git_sha": "fb67250", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n152_k10", "path": "cells/cell_mnemoverse_http_locomo_conv26_n152_k10.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.2961, "judge_accuracy_by_judge": { "mnemoverse": 0.296053, "mem0": 0.427632, "mem0-4o": 0.453947, "strict": 0.125 }, "cost_usd_total": 0.0263, "wallclock_s": 1864.82, "git_sha": "45e1a23", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n152_k100", "path": "cells/cell_mnemoverse_http_locomo_conv26_n152_k100.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5263, "judge_accuracy_by_judge": { "mnemoverse": 0.526316, "mem0": 0.697368, "mem0-4o": 0.782895, "strict": 0.282895 }, "cost_usd_total": 0.2124, "wallclock_s": 3135.89, "git_sha": "6e39d25", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n152_k20", "path": "cells/cell_mnemoverse_http_locomo_conv26_n152_k20.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.3618, "judge_accuracy_by_judge": { "mnemoverse": 0.361842, "mem0": 0.506579, "mem0-4o": 0.565789, "strict": 0.151316 }, "cost_usd_total": 0.0456, "wallclock_s": 2171.13, "git_sha": "45e1a23", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n152_k200", "path": "cells/cell_mnemoverse_http_locomo_conv26_n152_k200.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5395, "judge_accuracy_by_judge": { "mnemoverse": 0.539474, "mem0": 0.717105, "mem0-4o": 0.789474, "strict": 0.315789 }, "cost_usd_total": 0.4133, "wallclock_s": 3582.69, "git_sha": "6e39d25", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv26_n152_k50", "path": "cells/cell_mnemoverse_http_locomo_conv26_n152_k50.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.4079, "judge_accuracy_by_judge": { "mnemoverse": 0.407895, "mem0": 0.578947, "mem0-4o": 0.657895, "strict": 0.217105 }, "cost_usd_total": 0.1039, "wallclock_s": 2547.39, "git_sha": "45e1a23", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n152_k10", "path": "cells/cell_naked_cosine_locomo_conv26_n152_k10.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5921, "judge_accuracy_by_judge": { "mnemoverse": 0.592105, "mem0": 0.769737, "mem0-4o": 0.815789, "strict": 0.282895 }, "cost_usd_total": 0.1744, "wallclock_s": 1551.49, "git_sha": "1cc6512", "eval_path": "asymmetric_v1", "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08", "historic-2026-06-04" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n152_k100", "path": "cells/cell_naked_cosine_locomo_conv26_n152_k100.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6645, "judge_accuracy_by_judge": { "mnemoverse": 0.664474, "mem0": 0.809211, "mem0-4o": 0.881579, "strict": 0.407895 }, "cost_usd_total": 0.3237, "wallclock_s": 1642.39, "git_sha": "1cc6512", "eval_path": "asymmetric_v1", "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08", "historic-2026-06-04" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n152_k20", "path": "cells/cell_naked_cosine_locomo_conv26_n152_k20.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5987, "judge_accuracy_by_judge": { "mnemoverse": 0.598684, "mem0": 0.776316, "mem0-4o": 0.855263, "strict": 0.335526 }, "cost_usd_total": 0.1952, "wallclock_s": 1529.98, "git_sha": "1cc6512", "eval_path": "asymmetric_v1", "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08", "historic-2026-06-04" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n152_k200", "path": "cells/cell_naked_cosine_locomo_conv26_n152_k200.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.7039, "judge_accuracy_by_judge": { "mnemoverse": 0.703947, "mem0": 0.842105, "mem0-4o": 0.901316, "strict": 0.375 }, "cost_usd_total": 0.4677, "wallclock_s": 1336.55, "git_sha": "1cc6512", "eval_path": "asymmetric_v1", "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08", "historic-2026-06-04" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv26_n152_k50", "path": "cells/cell_naked_cosine_locomo_conv26_n152_k50.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "mem0": 0.809211, "mem0-4o": 0.881579, "strict": 0.361842 }, "cost_usd_total": 0.2445, "wallclock_s": 1510.18, "git_sha": "1cc6512", "eval_path": "asymmetric_v1", "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08", "historic-2026-06-04" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k10", "path": "cells/cell_supermemory_locomo_conv26_n152_k10.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.3421, "judge_accuracy_by_judge": { "mnemoverse": 0.342105, "mem0": 0.421053, "mem0-4o": 0.473684, "strict": 0.190789 }, "cost_usd_total": 0.0201, "wallclock_s": 1422.42, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k100", "path": "cells/cell_supermemory_locomo_conv26_n152_k100.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6382, "judge_accuracy_by_judge": { "mnemoverse": 0.638158, "mem0": 0.769737, "mem0-4o": 0.828947, "strict": 0.335526 }, "cost_usd_total": 0.0845, "wallclock_s": 1430.49, "git_sha": "fb67250", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k20", "path": "cells/cell_supermemory_locomo_conv26_n152_k20.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.4211, "judge_accuracy_by_judge": { "mnemoverse": 0.421053, "mem0": 0.532895, "mem0-4o": 0.546053, "strict": 0.236842 }, "cost_usd_total": 0.0439, "wallclock_s": 1455.25, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k200", "path": "cells/cell_supermemory_locomo_conv26_n152_k200.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6447, "judge_accuracy_by_judge": { "mnemoverse": 0.644737, "mem0": 0.796053, "mem0-4o": 0.835526, "strict": 0.348684 }, "cost_usd_total": 0.0843, "wallclock_s": 1422.59, "git_sha": "fb67250", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv26_n152_k50", "path": "cells/cell_supermemory_locomo_conv26_n152_k50.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6513, "judge_accuracy_by_judge": { "mnemoverse": 0.651316, "mem0": 0.789474, "mem0-4o": 0.822368, "strict": 0.348684 }, "cost_usd_total": 0.0843, "wallclock_s": 1563.56, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv26_n152_k10", "path": "cells/cell_zep_locomo_conv26_n152_k10.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.2171, "judge_accuracy_by_judge": { "mnemoverse": 0.217105, "mem0": 0.296053, "mem0-4o": 0.328947, "strict": 0.085526 }, "cost_usd_total": 0.0117, "wallclock_s": 1147.24, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv26_n152_k20", "path": "cells/cell_zep_locomo_conv26_n152_k20.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.3553, "judge_accuracy_by_judge": { "mnemoverse": 0.355263, "mem0": 0.480263, "mem0-4o": 0.546053, "strict": 0.125 }, "cost_usd_total": 0.0167, "wallclock_s": 1133.91, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv26_n152_k50", "path": "cells/cell_zep_locomo_conv26_n152_k50.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv26_n152", "benchmark_scope_n": 152, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.4803, "judge_accuracy_by_judge": { "mnemoverse": 0.480263, "mem0": 0.638158, "mem0-4o": 0.684211, "strict": 0.190789 }, "cost_usd_total": 0.0318, "wallclock_s": 1202.61, "git_sha": "70b3ed2", "eval_path": null, "known_asymmetries": null, "asymmetry_inventory_url": null, "tags": [ "baseline-2026-06-08" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k10", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k10.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5947, "judge_accuracy_by_judge": { "mnemoverse": 0.594737, "strict": 0.373684, "mem0": 0.684211, "mem0-4o": 0.705263 }, "cost_usd_total": 0.1797, "wallclock_s": 2288.26, "git_sha": "5bd54cb", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k100", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k100.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "strict": 0.4, "mem0": 0.721053, "mem0-4o": 0.757895 }, "cost_usd_total": 0.304, "wallclock_s": 2330.49, "git_sha": "592b530", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k20", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k20.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6474, "judge_accuracy_by_judge": { "mnemoverse": 0.647368, "strict": 0.405263, "mem0": 0.731579, "mem0-4o": 0.752632 }, "cost_usd_total": 0.1934, "wallclock_s": 1640.96, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k200", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k200.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6737, "judge_accuracy_by_judge": { "mnemoverse": 0.673684, "strict": 0.415789, "mem0": 0.752632, "mem0-4o": 0.778947 }, "cost_usd_total": 0.4377, "wallclock_s": 1751.48, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mem0_v3_cloud_locomo_conv47_n190_k50", "path": "cells/cell_mem0_v3_cloud_locomo_conv47_n190_k50.json", "system": "mem0_v3_cloud", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6368, "judge_accuracy_by_judge": { "mnemoverse": 0.636842, "strict": 0.384211, "mem0": 0.715789, "mem0-4o": 0.747368 }, "cost_usd_total": 0.2352, "wallclock_s": 1676, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k10", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k10.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5895, "judge_accuracy_by_judge": { "mnemoverse": 0.589474, "strict": 0.457895, "mem0": 0.752632, "mem0-4o": 0.778947 }, "cost_usd_total": 0.213, "wallclock_s": 2320.37, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.22, "10": 0.257778 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.313725, "10": 0.338235 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.116667, "10": 0.183333 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.240964, "10": 0.283133 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10 ], "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k100", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k100.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6421, "judge_accuracy_by_judge": { "mnemoverse": 0.642105, "strict": 0.473684, "mem0": 0.784211, "mem0-4o": 0.821053 }, "cost_usd_total": 0.5161, "wallclock_s": 5247.8, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.212222, "10": 0.257778, "20": 0.286667, "50": 0.302222, "100": 0.311667 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0, "50": 0, "100": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.279412, "10": 0.338235, "20": 0.352941, "50": 0.362745, "100": 0.362745 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.158333, "20": 0.2, "50": 0.2, "100": 0.270833 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.23494, "10": 0.289157, "20": 0.325301, "50": 0.349398, "100": 0.349398 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100 ], "cell_native_k": 100 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k20", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k20.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6105, "judge_accuracy_by_judge": { "mnemoverse": 0.610526, "strict": 0.436842, "mem0": 0.768421, "mem0-4o": 0.821053 }, "cost_usd_total": 0.2598, "wallclock_s": 3068.4, "git_sha": "592b530", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.216667, "10": 0.257778, "20": 0.283333 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.284314, "10": 0.338235, "20": 0.338235 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.183333, "20": 0.2 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.240964, "10": 0.283133, "20": 0.325301 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20 ], "cell_native_k": 20 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k200", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k200.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6737, "judge_accuracy_by_judge": { "mnemoverse": 0.673684, "strict": 0.489474, "mem0": 0.810526, "mem0-4o": 0.857895 }, "cost_usd_total": 0.7767, "wallclock_s": 6113.16, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.213333, "10": 0.253333, "20": 0.286667, "50": 0.3, "100": 0.310556, "200": 0.316667 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0, "50": 0, "100": 0, "200": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.284314, "10": 0.338235, "20": 0.352941, "50": 0.352941, "100": 0.362745, "200": 0.377451 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.175, "20": 0.2, "50": 0.2, "100": 0.2625, "200": 0.283333 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.23494, "10": 0.277108, "20": 0.325301, "50": 0.349398, "100": 0.349398, "200": 0.349398 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100, 200 ], "cell_native_k": 200 }, "comparative": true }, { "cell_id": "cell_mnemoverse_http_locomo_conv47_n190_k50", "path": "cells/cell_mnemoverse_http_locomo_conv47_n190_k50.json", "system": "mnemoverse_http", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6105, "judge_accuracy_by_judge": { "mnemoverse": 0.610526, "strict": 0.452632, "mem0": 0.763158, "mem0-4o": 0.805263 }, "cost_usd_total": 0.3593, "wallclock_s": 3791.89, "git_sha": "afda002", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-012", "ASYM-014", "ASYM-022", "ASYM-026", "ASYM-027", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "lower_bound", "recall_at_k": { "overall_recall_at_k": { "5": 0.213333, "10": 0.254444, "20": 0.286667, "50": 0.302222 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0, "10": 0, "20": 0, "50": 0 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.269608, "10": 0.338235, "20": 0.352941, "50": 0.362745 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.141667, "10": 0.183333, "20": 0.2, "50": 0.2 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.240964, "10": 0.277108, "20": 0.325301, "50": 0.349398 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50 ], "cell_native_k": 50 }, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k10", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k10.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5105, "judge_accuracy_by_judge": { "mnemoverse": 0.510526, "strict": 0.352632, "mem0": 0.657895, "mem0-4o": 0.721053 }, "cost_usd_total": 0.1979, "wallclock_s": 2112.5, "git_sha": "73083d4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k100", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k100.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6263, "judge_accuracy_by_judge": { "mnemoverse": 0.626316, "strict": 0.463158, "mem0": 0.763158, "mem0-4o": 0.815789 }, "cost_usd_total": 0.4579, "wallclock_s": 1866.93, "git_sha": "f6c536b", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k20", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k20.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5842, "judge_accuracy_by_judge": { "mnemoverse": 0.584211, "strict": 0.447368, "mem0": 0.726316, "mem0-4o": 0.794737 }, "cost_usd_total": 0.227, "wallclock_s": 1901.93, "git_sha": "73083d4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k200", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k200.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6526, "judge_accuracy_by_judge": { "mnemoverse": 0.652632, "strict": 0.463158, "mem0": 0.778947, "mem0-4o": 0.836842 }, "cost_usd_total": 0.7003, "wallclock_s": 2082.32, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_mnemoverse_locomo_conv47_n190_k50", "path": "cells/cell_mnemoverse_locomo_conv47_n190_k50.json", "system": "mnemoverse", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6579, "judge_accuracy_by_judge": { "mnemoverse": 0.657895, "strict": 0.452632, "mem0": 0.8, "mem0-4o": 0.831579 }, "cost_usd_total": 0.3254, "wallclock_s": 1956.73, "git_sha": "73083d4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-019", "ASYM-022", "ASYM-028", "ASYM-030" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k10", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k10.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5421, "judge_accuracy_by_judge": { "mnemoverse": 0.542105, "strict": 0.405263, "mem0": 0.678947, "mem0-4o": 0.742105 }, "cost_usd_total": 0.1878, "wallclock_s": 1444.19, "git_sha": "19a16e4", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10 ], "cell_native_k": 10 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k100", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k100.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6053, "judge_accuracy_by_judge": { "mnemoverse": 0.605263, "strict": 0.452632, "mem0": 0.805263, "mem0-4o": 0.852632 }, "cost_usd_total": 0.3739, "wallclock_s": 1383.08, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805, "50": 0.89, "100": 0.933889 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385, "50": 0.692308, "100": 0.692308 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922, "50": 0.872549, "100": 0.901961 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833, "50": 0.741667, "100": 0.870833 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614, "50": 0.963855, "100": 1 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100 ], "cell_native_k": 100 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k20", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k20.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.5789, "judge_accuracy_by_judge": { "mnemoverse": 0.578947, "strict": 0.405263, "mem0": 0.736842, "mem0-4o": 0.789474 }, "cost_usd_total": 0.2093, "wallclock_s": 1583.91, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20 ], "cell_native_k": 20 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k200", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k200.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 200, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6684, "judge_accuracy_by_judge": { "mnemoverse": 0.668421, "strict": 0.457895, "mem0": 0.821053, "mem0-4o": 0.857895 }, "cost_usd_total": 0.5811, "wallclock_s": 1387.42, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805, "50": 0.89, "100": 0.933889, "200": 0.98 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385, "50": 0.692308, "100": 0.692308, "200": 0.923077 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922, "50": 0.872549, "100": 0.901961, "200": 0.97549 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833, "50": 0.741667, "100": 0.870833, "200": 0.941667 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614, "50": 0.963855, "100": 1, "200": 1 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50, 100, 200 ], "cell_native_k": 200 }, "comparative": true }, { "cell_id": "cell_naked_cosine_locomo_conv47_n190_k50", "path": "cells/cell_naked_cosine_locomo_conv47_n190_k50.json", "system": "naked_cosine", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "strict": 0.442105, "mem0": 0.789474, "mem0-4o": 0.815789 }, "cost_usd_total": 0.271, "wallclock_s": 1420.82, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-014", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "exact", "recall_at_k": { "overall_recall_at_k": { "5": 0.674444, "10": 0.733889, "20": 0.805, "50": 0.89 }, "by_category_recall_at_k": { "open_domain": { "n": 13, "recall_at_k": { "5": 0.538462, "10": 0.576923, "20": 0.615385, "50": 0.692308 } }, "temporal": { "n": 34, "recall_at_k": { "5": 0.730392, "10": 0.803922, "20": 0.803922, "50": 0.872549 } }, "multi_hop": { "n": 20, "recall_at_k": { "5": 0.266667, "10": 0.3125, "20": 0.520833, "50": 0.741667 } }, "single_hop": { "n": 83, "recall_at_k": { "5": 0.771084, "10": 0.831325, "20": 0.903614, "50": 0.963855 } } }, "n_questions": 190, "n_questions_with_evidence_eval": 150, "n_questions_with_placeholder_retrieval": 0, "ks_computed": [ 5, 10, 20, 50 ], "cell_native_k": 50 }, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k10", "path": "cells/cell_supermemory_locomo_conv47_n190_k10.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6316, "judge_accuracy_by_judge": { "mnemoverse": 0.631579, "strict": 0.378947, "mem0": 0.731579, "mem0-4o": 0.757895 }, "cost_usd_total": 0.1812, "wallclock_s": 2088.17, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k100", "path": "cells/cell_supermemory_locomo_conv47_n190_k100.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 100, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.7211, "judge_accuracy_by_judge": { "mnemoverse": 0.721053, "strict": 0.489474, "mem0": 0.794737, "mem0-4o": 0.8 }, "cost_usd_total": 0.3198, "wallclock_s": 2158.37, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k20", "path": "cells/cell_supermemory_locomo_conv47_n190_k20.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.6895, "judge_accuracy_by_judge": { "mnemoverse": 0.689474, "strict": 0.415789, "mem0": 0.747368, "mem0-4o": 0.768421 }, "cost_usd_total": 0.1954, "wallclock_s": 1788.1, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_supermemory_locomo_conv47_n190_k50", "path": "cells/cell_supermemory_locomo_conv47_n190_k50.json", "system": "supermemory", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.7053, "judge_accuracy_by_judge": { "mnemoverse": 0.705263, "strict": 0.436842, "mem0": 0.784211, "mem0-4o": 0.794737 }, "cost_usd_total": 0.2405, "wallclock_s": 1753.42, "git_sha": "844522a", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-020", "ASYM-022", "ASYM-028", "ASYM-029" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv47_n190_k10", "path": "cells/cell_zep_locomo_conv47_n190_k10.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 10, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.0895, "judge_accuracy_by_judge": { "mnemoverse": 0.089474, "strict": 0.068421, "mem0": 0.121053, "mem0-4o": 0.142105 }, "cost_usd_total": 0.1716, "wallclock_s": 1322.58, "git_sha": "6c8cd98", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv47_n190_k20", "path": "cells/cell_zep_locomo_conv47_n190_k20.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 20, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.1579, "judge_accuracy_by_judge": { "mnemoverse": 0.157895, "strict": 0.1, "mem0": 0.231579, "mem0-4o": 0.242105 }, "cost_usd_total": 0.1788, "wallclock_s": 1430.35, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true }, { "cell_id": "cell_zep_locomo_conv47_n190_k50", "path": "cells/cell_zep_locomo_conv47_n190_k50.json", "system": "zep", "benchmark": "locomo", "benchmark_scope": "conv47_n190", "benchmark_scope_n": 190, "role": "headline", "k": 50, "reader_model": "gpt-5-mini", "primary_judge": "mnemoverse", "headline_metric": 0.2316, "judge_accuracy_by_judge": { "mnemoverse": 0.231579, "strict": 0.152632, "mem0": 0.326316, "mem0-4o": 0.357895 }, "cost_usd_total": 0.1972, "wallclock_s": 1737.36, "git_sha": "44ddb9f", "eval_path": "symmetric_v1", "known_asymmetries": [ "ASYM-011", "ASYM-013", "ASYM-014", "ASYM-016", "ASYM-022", "ASYM-028" ], "asymmetry_inventory_url": "https://graph.mnemoverse.com/benchmarks/methodology/asymmetric-v1", "tags": [ "matrix-v2-conv47-2026-06-11" ], "recall_provenance": "n_a", "recall_at_k": null, "comparative": true } ], "system_labels": { "mem0_v3_cloud": "Mem0", "mnemoverse": "Mnemoverse", "mnemoverse_engine": "Mnemoverse", "mnemoverse_http": "Mnemoverse", "naked_cosine": "Naked cosine (floor)", "supermemory": "Supermemory", "zep": "Zep" }, "quarantine": { "render_contract": "Render as 'not measurable' cards per docs/RENDER_CONTRACT.md rule 1; never as zeros, never ranked. Superseded entries are NOT rendered at all — the good re-run (superseded_by_git_sha) is the live cell; this entry only stops a basename-keyed gate from excluding it.", "n_entries": 9, "excluded_cell_ids": [] } }