{
  "study": "how-often-is-a-code-check-right",
  "exported": "2026-10-09",
  "source": {
    "repository": "Vallit-Network/vallit-app",
    "commit": "09c2faa9a66543fc7b51dabdb3583dbdcbdc127f",
    "files": [
      {
        "path": "bench/source/RESULTS.md",
        "sha256": "5d1561899d0faf6234ff02a8ed2223934a69e3c1956caac31291767306aaeb29"
      },
      {
        "path": "bench/source/results/holdout-a-v1.json",
        "sha256": "f82bd8ff122fbcad08b16444a8fe8664d0105f967c5304332cfd0e60bef95e21"
      },
      {
        "path": "bench/source/results/holdout-b-v2.json",
        "sha256": "972bab2f6a0a6f531bd4b5eb93ecdbb1e34d80a5c07eade2aa1cb65bcccbd8f1"
      },
      {
        "path": "bench/source/results/holdout-c-v3.json",
        "sha256": "7a2d0e0a0cc03b5f447f334394456383350a340ddc7df98d1f560649e86d6daf"
      },
      {
        "path": "bench/source/results/holdout-d-v4-static.json",
        "sha256": "4bca955d48eef38e63fc5351238a7490367fe12159f17f0d02b4a3d26843d709"
      },
      {
        "path": "bench/source/results/holdout-d-v4-verified.json",
        "sha256": "4ed0d4b2db5d14a9a2fc89afb7f88faf2100a3bff0761d29995e03885f5ae24a"
      },
      {
        "path": "bench/source/results/holdout-e-v5-static.json",
        "sha256": "e4ac967915d30daf2c36a677c41f4662b20ca88a120964e2bb720ce13bf6ea4f"
      },
      {
        "path": "bench/source/results/holdout-e-v5-verified.json",
        "sha256": "4777acf7c951dcd9dd492a2d44280c1f3edbe425502dc98ab5c4a98ec9dc8474"
      },
      {
        "path": "docs/audits/deep-detections-2026-09-23/holdout-a-regression.json",
        "sha256": "f681733605bd0994323c4364db5e811b1d326fb80f12efed02799f56514881ed"
      },
      {
        "path": "docs/audits/deep-detections-2026-09-23/holdout-b-regression.json",
        "sha256": "9268d5efc75b217b63862d79c11123afc673fc5096d0798eed6b382aaef03147"
      },
      {
        "path": "docs/audits/deep-detections-2026-09-23/holdout-c-regression.json",
        "sha256": "42cffaf5e4afd2685c61d46ca2af0fe11ad7b46d4c60526f2e78f96215f19e59"
      },
      {
        "path": "docs/audits/deep-detections-2026-09-23/holdout-d-regression.json",
        "sha256": "31a3d64ad7bddf344fdf4c63afe580ab62e5de5fae3474b872834d1385ac3594"
      },
      {
        "path": "docs/audits/deep-detections-2026-09-23/holdout-e-regression.json",
        "sha256": "05ee337b40a6c81dcd25865d9b4b17aa190e9d54979f80906615a50defb36f28"
      },
      {
        "path": "bench/source/results/jev-screen-per-check.json",
        "sha256": "5ff79ab5a465d3a2d745ce80416193eacb735e3e9ce7dd25f10c652125819871"
      },
      {
        "path": "docs/audits/deep-detections-2026-09-23/identity-ai-final-verified.json",
        "sha256": "266a60f4d82f0f1f804ed3015f03b24f3c8b589b99f6269a029c4e3ce60bf0f1"
      },
      {
        "path": "bench/source/results/fresh-repos-v3.raw.json",
        "sha256": "645022649de0802f4da0104a05aad2e523c1d1683f538c66bb70951b64baae4e"
      },
      {
        "path": "bench/source/results/fresh-repos-v3.adjudication-dub.json",
        "sha256": "ace28bc52880dc28b87d8a5bd06c7a6080d40f03c50d1adc93ee78262442d1de"
      },
      {
        "path": "bench/source/results/fresh-repos-v3.adjudication-langfuse-rallly-dokploy.json",
        "sha256": "d4a5c6792b3a92f2bd1ff68de76701183f4cc4c96e113b637185b76e120df297"
      },
      {
        "path": "bench/source/results/fresh-repos-v3.adjudication-lobe-unkey-chatbot-starter.json",
        "sha256": "ec623e5506aeeddcc09c7d731676f337659801ae509422e283d69f10bcafdf91"
      },
      {
        "path": "bench/source/results/unseen-repos-v4.raw.json",
        "sha256": "8600a7fa8bbdb0fa458570a30c018e34864a4702f90c922ab7ff15473fe5d734"
      },
      {
        "path": "bench/source/results/unseen-repos-v4.adjudication.json",
        "sha256": "0edfd7d1208534fe75e91285190b8a0bf0f101f44ff4d2155c5d0a3bc8e6547c"
      },
      {
        "path": "bench/source/results/unseen-repos-v4.commits.txt",
        "sha256": "e7047c53de9efcd80901ddcaff06963d0f0c75c97db2eadc333faa6d919a41ef"
      },
      {
        "path": "bench/source/results/unseen3-repos-v5.raw.json",
        "sha256": "df1183855c44df9f87cc845a3a51010f8db9509e1c0a62dd0fdf6fc1f6d8c89d"
      },
      {
        "path": "bench/source/results/unseen3-repos-v5.adjudication.json",
        "sha256": "88243e5632308e645c5a398358f7c8b8bb75d45fe9e6a3f91c2b135a88e53071"
      },
      {
        "path": "bench/source/results/unseen3-repos-v5.commits.txt",
        "sha256": "ec5f0f60ea311e7aba58c42b8929645f679b429b74097f8c1de1cef45602769a"
      },
      {
        "path": "bench/source/results/unseen3-repos-v5.audit.json",
        "sha256": "eac28f93d5a8155ade3c1eb38a0f6892a0b9d8ae4468d6a53628873fc24f96f4"
      }
    ]
  },
  "lab": {
    "cases": 560,
    "filesPerCase": {
      "min": 1,
      "max": 9
    },
    "sets": [
      {
        "set": "A",
        "detector": "v1",
        "cases": 80,
        "detectorSha256": "ad6d0a02abc4449943e8e7adca34ccfd08412a3e8f8a3e807cbc8ed3c1c07241",
        "ranAt": "2026-09-17T01:41:24.595Z",
        "static": {
          "tp": 29,
          "fp": 4,
          "fn": 16,
          "precision": 0.878788,
          "precisionCI": [
            0.726742,
            0.951839
          ],
          "recall": 0.644444,
          "recallCI": [
            0.498355,
            0.767811
          ],
          "vulnerable": 44,
          "vulnerableFound": 30,
          "vulnerableRight": 29,
          "secure": 36,
          "secureFlagged": 1
        },
        "verified": null
      },
      {
        "set": "B",
        "detector": "v2",
        "cases": 120,
        "detectorSha256": "ee8a916699df08183be48a29ae955fa8247e3e4842f8bfa5a792b7097d9fcdd2",
        "ranAt": "2026-09-17T01:56:05.478Z",
        "static": {
          "tp": 54,
          "fp": 6,
          "fn": 18,
          "precision": 0.9,
          "precisionCI": [
            0.798503,
            0.953358
          ],
          "recall": 0.75,
          "recallCI": [
            0.639063,
            0.835611
          ],
          "vulnerable": 66,
          "vulnerableFound": 55,
          "vulnerableRight": 52,
          "secure": 54,
          "secureFlagged": 2
        },
        "verified": null
      },
      {
        "set": "C",
        "detector": "v3",
        "cases": 120,
        "detectorSha256": "ef54bac8fefc5c13d06c8c9ad5f45d275463f6690cf4cee6e54ca191bbf1929e",
        "ranAt": "2026-09-17T02:10:32.273Z",
        "static": {
          "tp": 61,
          "fp": 5,
          "fn": 12,
          "precision": 0.924242,
          "precisionCI": [
            0.834606,
            0.967208
          ],
          "recall": 0.835616,
          "recallCI": [
            0.734287,
            0.903388
          ],
          "vulnerable": 66,
          "vulnerableFound": 56,
          "vulnerableRight": 55,
          "secure": 54,
          "secureFlagged": 2
        },
        "verified": null
      },
      {
        "set": "D",
        "detector": "v4",
        "cases": 120,
        "detectorSha256": "65938b224e890ac93a2ec0d24e0cb79bddf7b3c249b06f71fd41c380edfae9be",
        "ranAt": "2026-09-17T07:01:38.705Z",
        "static": {
          "tp": 50,
          "fp": 5,
          "fn": 24,
          "precision": 0.909091,
          "precisionCI": [
            0.804223,
            0.960542
          ],
          "recall": 0.675676,
          "recallCI": [
            0.562651,
            0.771361
          ],
          "vulnerable": 66,
          "vulnerableFound": 52,
          "vulnerableRight": 49,
          "secure": 54,
          "secureFlagged": 2
        },
        "verified": {
          "tp": 50,
          "fp": 3,
          "fn": 24,
          "precision": 0.943396,
          "precisionCI": [
            0.846296,
            0.980564
          ],
          "recall": 0.675676,
          "recallCI": [
            0.562651,
            0.771361
          ],
          "vulnerable": 66,
          "vulnerableFound": 51,
          "vulnerableRight": 49,
          "secure": 54,
          "secureFlagged": 1,
          "models": [
            "claude-haiku-4-5-20251001",
            "claude-sonnet-5"
          ],
          "calls": 150,
          "costUsd": 1.8233,
          "falseAlarmsRemoved": 2,
          "realProblemsLost": 0
        }
      },
      {
        "set": "E",
        "detector": "v5",
        "cases": 120,
        "detectorSha256": "5c013d91ab6baae7c59e16213894b47938dadec02e7cdb26e59d8c0db5c56da2",
        "ranAt": "2026-09-17T08:10:43.076Z",
        "static": {
          "tp": 59,
          "fp": 11,
          "fn": 15,
          "precision": 0.842857,
          "precisionCI": [
            0.740115,
            0.909925
          ],
          "recall": 0.797297,
          "recallCI": [
            0.69212,
            0.87313
          ],
          "vulnerable": 66,
          "vulnerableFound": 56,
          "vulnerableRight": 54,
          "secure": 54,
          "secureFlagged": 5
        },
        "verified": {
          "tp": 59,
          "fp": 7,
          "fn": 15,
          "precision": 0.893939,
          "precisionCI": [
            0.796875,
            0.947667
          ],
          "recall": 0.797297,
          "recallCI": [
            0.69212,
            0.87313
          ],
          "vulnerable": 66,
          "vulnerableFound": 56,
          "vulnerableRight": 54,
          "secure": 54,
          "secureFlagged": 2,
          "models": [
            "claude-haiku-4-5-20251001",
            "claude-sonnet-5"
          ],
          "calls": 118,
          "costUsd": 1.6488,
          "allRuns": {
            "calls": 199,
            "costUsd": 2.79,
            "note": "a first run was cut short by a rate limit, and its 43 failed verdicts were verified again in a second run"
          },
          "falseAlarmsRemoved": 4,
          "realProblemsLost": 0
        }
      }
    ],
    "rules": [
      {
        "check": "access-control",
        "rule": "client-chosen-owner",
        "tp": 28,
        "fp": 2,
        "fn": 20,
        "recall": 0.583333,
        "recallCI": [
          0.442811,
          0.711505
        ],
        "precision": 0.933333
      },
      {
        "check": "access-control",
        "rule": "missing-ownership",
        "tp": 42,
        "fp": 12,
        "fn": 9,
        "recall": 0.823529,
        "recallCI": [
          0.697456,
          0.904277
        ],
        "precision": 0.777778
      },
      {
        "check": "access-control",
        "rule": "privilege-escalation",
        "tp": 28,
        "fp": 1,
        "fn": 11,
        "recall": 0.717949,
        "recallCI": [
          0.562244,
          0.834567
        ],
        "precision": 0.965517
      },
      {
        "check": "access-control",
        "rule": "unauthenticated-write",
        "tp": 36,
        "fp": 7,
        "fn": 7,
        "recall": 0.837209,
        "recallCI": [
          0.700273,
          0.918835
        ],
        "precision": 0.837209
      },
      {
        "check": "payment-integrity",
        "rule": "cancel-keeps-access",
        "tp": 10,
        "fp": 2,
        "fn": 9,
        "recall": 0.526316,
        "recallCI": [
          0.317075,
          0.726705
        ],
        "precision": 0.833333
      },
      {
        "check": "payment-integrity",
        "rule": "client-set-price",
        "tp": 24,
        "fp": 3,
        "fn": 5,
        "recall": 0.827586,
        "recallCI": [
          0.654512,
          0.924023
        ],
        "precision": 0.888889
      },
      {
        "check": "payment-integrity",
        "rule": "duplicate-grant",
        "tp": 15,
        "fp": 3,
        "fn": 10,
        "recall": 0.6,
        "recallCI": [
          0.407391,
          0.765969
        ],
        "precision": 0.833333
      },
      {
        "check": "payment-integrity",
        "rule": "unpaid-upgrade",
        "tp": 26,
        "fp": 0,
        "fn": 10,
        "recall": 0.722222,
        "recallCI": [
          0.560072,
          0.841518
        ],
        "precision": 1
      },
      {
        "check": "payment-integrity",
        "rule": "unverified-webhook",
        "tp": 25,
        "fp": 1,
        "fn": 2,
        "recall": 0.925926,
        "recallCI": [
          0.7663,
          0.979446
        ],
        "precision": 0.961538
      },
      {
        "check": "payment-integrity",
        "rule": "webhook-body-parsed",
        "tp": 19,
        "fp": 0,
        "fn": 2,
        "recall": 0.904762,
        "recallCI": [
          0.710854,
          0.973482
        ],
        "precision": 1
      }
    ],
    "later": [
      {
        "set": "A",
        "detectorSha256": "b7b3ec07c4f17d7aab9b5282f0e5b9477c3e8eb765032ae133fefd6ea1fba9ad",
        "ranAt": "2026-09-23T11:10:17.722Z",
        "first": {
          "precision": 0.878788,
          "recall": 0.644444
        },
        "later": {
          "tp": 49,
          "fp": 0,
          "fn": 0,
          "precision": 1,
          "recall": 1
        }
      },
      {
        "set": "B",
        "detectorSha256": "b7b3ec07c4f17d7aab9b5282f0e5b9477c3e8eb765032ae133fefd6ea1fba9ad",
        "ranAt": "2026-09-23T11:10:22.484Z",
        "first": {
          "precision": 0.9,
          "recall": 0.75
        },
        "later": {
          "tp": 75,
          "fp": 0,
          "fn": 0,
          "precision": 1,
          "recall": 1
        }
      },
      {
        "set": "C",
        "detectorSha256": "b7b3ec07c4f17d7aab9b5282f0e5b9477c3e8eb765032ae133fefd6ea1fba9ad",
        "ranAt": "2026-09-23T11:10:25.784Z",
        "first": {
          "precision": 0.924242,
          "recall": 0.835616
        },
        "later": {
          "tp": 72,
          "fp": 2,
          "fn": 2,
          "precision": 0.972973,
          "recall": 0.972973
        }
      },
      {
        "set": "D",
        "detectorSha256": "b7b3ec07c4f17d7aab9b5282f0e5b9477c3e8eb765032ae133fefd6ea1fba9ad",
        "ranAt": "2026-09-23T11:10:28.251Z",
        "first": {
          "precision": 0.909091,
          "recall": 0.675676
        },
        "later": {
          "tp": 69,
          "fp": 4,
          "fn": 5,
          "precision": 0.945205,
          "recall": 0.932432
        }
      },
      {
        "set": "E",
        "detectorSha256": "b7b3ec07c4f17d7aab9b5282f0e5b9477c3e8eb765032ae133fefd6ea1fba9ad",
        "ranAt": "2026-09-23T11:10:30.781Z",
        "first": {
          "precision": 0.842857,
          "recall": 0.797297
        },
        "later": {
          "tp": 59,
          "fp": 11,
          "fn": 15,
          "precision": 0.842857,
          "recall": 0.797297
        }
      }
    ],
    "rerun": {
      "appCommit": "0211efb5806d36fa02ed34c2e3635a742cfbbf70",
      "detectorSha256": "5f5d69a5bdcbc32259d905c28281527cb03fb7833d76a13c88dd7d917c1bf252",
      "date": "2026-09-24",
      "sets": [
        {
          "set": "A",
          "tp": 49,
          "fp": 0,
          "fn": 0,
          "detectorSha256": "5f5d69a5bdcbc32259d905c28281527cb03fb7833d76a13c88dd7d917c1bf252",
          "ranAt": "2026-09-24T14:34:35.755Z",
          "matchesRecorded": true
        },
        {
          "set": "B",
          "tp": 75,
          "fp": 0,
          "fn": 0,
          "detectorSha256": "5f5d69a5bdcbc32259d905c28281527cb03fb7833d76a13c88dd7d917c1bf252",
          "ranAt": "2026-09-24T14:34:39.093Z",
          "matchesRecorded": true
        },
        {
          "set": "C",
          "tp": 72,
          "fp": 2,
          "fn": 2,
          "detectorSha256": "5f5d69a5bdcbc32259d905c28281527cb03fb7833d76a13c88dd7d917c1bf252",
          "ranAt": "2026-09-24T14:34:42.508Z",
          "matchesRecorded": true
        },
        {
          "set": "D",
          "tp": 69,
          "fp": 4,
          "fn": 5,
          "detectorSha256": "5f5d69a5bdcbc32259d905c28281527cb03fb7833d76a13c88dd7d917c1bf252",
          "ranAt": "2026-09-24T14:34:45.827Z",
          "matchesRecorded": true
        },
        {
          "set": "E",
          "tp": 59,
          "fp": 11,
          "fn": 15,
          "detectorSha256": "5f5d69a5bdcbc32259d905c28281527cb03fb7833d76a13c88dd7d917c1bf252",
          "ranAt": "2026-09-24T14:34:49.312Z",
          "matchesRecorded": true
        }
      ]
    }
  },
  "field": {
    "repositories": 30,
    "runs": [
      {
        "id": "v3",
        "detector": "v3",
        "mode": "static",
        "repositories": [
          {
            "repository": "Dokploy/dokploy",
            "commit": "853ca33659853093ee6a91af3719abdc4e1bae72",
            "files": 1208,
            "entryPoints": 449,
            "analysisMs": 734
          },
          {
            "repository": "dubinc/dub",
            "commit": "24c45d733c3c229846e258c024d1fccb93c38390",
            "files": 4421,
            "entryPoints": 736,
            "analysisMs": 2810
          },
          {
            "repository": "langfuse/langfuse",
            "commit": "53640f4ca74ba7d13a3bed0d4b1c88006609b0f5",
            "files": 5372,
            "entryPoints": 508,
            "analysisMs": 2452
          },
          {
            "repository": "linkwarden/linkwarden",
            "commit": "952ac4540657cae3a67c3ca59433899d2fda8374",
            "files": 649,
            "entryPoints": 54,
            "analysisMs": 274
          },
          {
            "repository": "lobehub/lobe-chat",
            "commit": "1adb625d4cf638483840237a159cf54621ade6a5",
            "files": 5315,
            "entryPoints": 1197,
            "analysisMs": 2302
          },
          {
            "repository": "lukevella/rallly",
            "commit": "bb8e45311ac82e004678d4dd5b8100a075b9564d",
            "files": 1189,
            "entryPoints": 104,
            "analysisMs": 377
          },
          {
            "repository": "mickasmt/next-saas-stripe-starter",
            "commit": "a78d130af7e04d0250d65c67f217976f7eb3adc2",
            "files": 191,
            "entryPoints": 7,
            "analysisMs": 60
          },
          {
            "repository": "midday-ai/midday",
            "commit": "51587319f26a0ffaa9dfccab1920373cb65689b7",
            "files": 2104,
            "entryPoints": 332,
            "analysisMs": 937
          },
          {
            "repository": "unkeyed/unkey",
            "commit": "d824c81cfbb6fc4d64ba834de4b6506fa8cd2e48",
            "files": 3302,
            "entryPoints": 234,
            "analysisMs": 1069
          },
          {
            "repository": "vercel/ai-chatbot",
            "commit": "c2f8235e1f3ea903ad8b7f61447c4f74164b5c58",
            "files": 157,
            "entryPoints": 23,
            "analysisMs": 88
          }
        ],
        "reported": 131,
        "real": 4,
        "unsure": 0,
        "precision": 0.030534,
        "precisionCI": [
          0.011937,
          0.075882
        ],
        "falseAlarmsByRule": {
          "unauthenticated-write": 61,
          "missing-ownership": 27,
          "client-chosen-owner": 27,
          "privilege-escalation": 7,
          "unpaid-upgrade": 2,
          "unverified-webhook": 1,
          "client-set-price": 1,
          "webhook-body-parsed": 1
        },
        "realByRule": {
          "missing-ownership": 2,
          "privilege-escalation": 1,
          "client-set-price": 1
        }
      },
      {
        "id": "v4",
        "detector": "v4",
        "detectorCommit": "0664a0c",
        "mode": "verified",
        "models": [
          "claude-haiku-4-5-20251001",
          "claude-sonnet-5"
        ],
        "repositories": [
          {
            "repository": "CapSoftware/Cap",
            "commit": "94bd7b46f8cdd79d6ce0328861525b2e3f4cdf56",
            "files": 2247,
            "entryPoints": 267,
            "analysisMs": 2012
          },
          {
            "repository": "Mail-0/Zero",
            "commit": "64c5480c341750578da0746f2db9ad84da686334",
            "files": 416,
            "entryPoints": 94,
            "analysisMs": 487
          },
          {
            "repository": "baptisteArno/typebot.io",
            "commit": "61056ff9a98082485add8111e901d3148ca358ef",
            "files": 2353,
            "entryPoints": 10,
            "analysisMs": 1095
          },
          {
            "repository": "calcom/cal.com",
            "commit": "6bc45298226f96ff79e0c070c8b2ce39727e8477",
            "files": 5646,
            "entryPoints": 205,
            "analysisMs": 1895
          },
          {
            "repository": "civitai/civitai",
            "commit": "563b0a7d4d7fde48916b0177ccca357c0d1fbdcb",
            "files": 5823,
            "entryPoints": 314,
            "analysisMs": 3194
          },
          {
            "repository": "formbricks/formbricks",
            "commit": "a54bd9b6781624e629f900bea270f2823a433c21",
            "files": 3898,
            "entryPoints": 277,
            "analysisMs": 2873
          },
          {
            "repository": "mckaywrigley/chatbot-ui",
            "commit": "81328b61d2a4ab597a7a057be70e785cf756d9f8",
            "files": 289,
            "entryPoints": 23,
            "analysisMs": 128
          },
          {
            "repository": "saasfly/saasfly",
            "commit": "49f7e28f69eae9cd9eed84221e13a3dbae87da67",
            "files": 227,
            "entryPoints": 15,
            "analysisMs": 74
          },
          {
            "repository": "unsend-dev/unsend",
            "commit": "bcf7e07e92b11f00fade54ff9a636ca7b5416b0c",
            "files": 489,
            "entryPoints": 113,
            "analysisMs": 267
          },
          {
            "repository": "vercel/platforms",
            "commit": "ec12e65709c8263dd3462c4d33261851b8a3157d",
            "files": 22,
            "entryPoints": 2,
            "analysisMs": 8
          }
        ],
        "candidates": 205,
        "likely": 25,
        "possible": 41,
        "rejected": 139,
        "judged": {
          "likely": {
            "judged": 25,
            "of": 25,
            "real": 9,
            "precision": 0.36,
            "precisionCI": [
              0.202477,
              0.554818
            ],
            "sample": "all"
          },
          "possible": {
            "judged": 41,
            "of": 41,
            "real": 2,
            "precision": 0.04878,
            "precisionCI": [
              0.013481,
              0.161393
            ],
            "sample": "all"
          },
          "rejected": {
            "judged": 31,
            "of": 139,
            "real": 0,
            "precision": 0,
            "precisionCI": null,
            "sample": "uneven",
            "byRepository": [
              {
                "repository": "formbricks/formbricks",
                "of": 99,
                "judged": 10,
                "real": 0,
                "precisionCI": [
                  0,
                  0.27754
                ]
              },
              {
                "repository": "CapSoftware/Cap",
                "of": 20,
                "judged": 10,
                "real": 0,
                "precisionCI": [
                  0,
                  0.27754
                ]
              },
              {
                "repository": "civitai/civitai",
                "of": 19,
                "judged": 10,
                "real": 0,
                "precisionCI": [
                  0,
                  0.27754
                ]
              },
              {
                "repository": "unsend-dev/unsend",
                "of": 1,
                "judged": 1,
                "real": 0,
                "precisionCI": [
                  0,
                  0.793457
                ]
              }
            ]
          }
        },
        "realByRule": {
          "missing-ownership": 5,
          "client-set-price": 4,
          "unauthenticated-write": 2
        },
        "calls": 573,
        "costUsd": 25.317541,
        "audit": null
      },
      {
        "id": "v5",
        "detector": "v5",
        "detectorCommit": "99fee29",
        "mode": "verified",
        "models": [
          "claude-haiku-4-5-20251001",
          "claude-sonnet-5"
        ],
        "repositories": [
          {
            "repository": "HeyPuter/puter",
            "commit": "bb97660ad70e740551b4c05a084b1e76c98cf330",
            "files": 1074,
            "entryPoints": 77,
            "analysisMs": 781
          },
          {
            "repository": "Openpanel-dev/openpanel",
            "commit": "3060ca10213693cf0385be2713c8743d16733a2b",
            "files": 1511,
            "entryPoints": 236,
            "analysisMs": 808
          },
          {
            "repository": "TryGhost/Ghost",
            "commit": "5f767ec890aa922c53b2420e054a5289fc2dcbe0",
            "files": 4892,
            "entryPoints": 69,
            "analysisMs": 1757
          },
          {
            "repository": "bigcapitalhq/bigcapital",
            "commit": "58114c18153fa8e168041e04d6d95f954d0eef53",
            "files": 5493,
            "entryPoints": 0,
            "analysisMs": 1510
          },
          {
            "repository": "karakeep-app/karakeep",
            "commit": "ccee905a4fa90041e7f77ca42ae175101c538ca2",
            "files": 997,
            "entryPoints": 222,
            "analysisMs": 416
          },
          {
            "repository": "mendableai/firecrawl",
            "commit": "f6774139f3d395905c4ad7982d06e984d88f6078",
            "files": 619,
            "entryPoints": 30,
            "analysisMs": 460
          },
          {
            "repository": "papra-hq/papra",
            "commit": "edbfe7ff5e68038ec1bdb7dbf9752ba6e074f159",
            "files": 1122,
            "entryPoints": 107,
            "analysisMs": 378
          },
          {
            "repository": "stack-auth/stack",
            "commit": "b4eddab8f0e95b6b93df5ba61e6abfe848a66689",
            "files": 2661,
            "entryPoints": 73,
            "analysisMs": 1290
          },
          {
            "repository": "vercel/next-forge",
            "commit": "f189de79ceef7c1ef69f61f12e272f99b4cdb699",
            "files": 359,
            "entryPoints": 18,
            "analysisMs": 124
          },
          {
            "repository": "webstudio-is/webstudio",
            "commit": "aa1587c100fc96a78aaa41489c242ce9d4ecd150",
            "files": 2334,
            "entryPoints": 87,
            "analysisMs": 1203
          }
        ],
        "candidates": 72,
        "likely": 0,
        "possible": 5,
        "rejected": 67,
        "judged": {
          "likely": {
            "judged": 0,
            "of": 0,
            "real": 0,
            "precision": null,
            "precisionCI": null,
            "sample": "all"
          },
          "possible": {
            "judged": 5,
            "of": 5,
            "real": 0,
            "precision": 0,
            "precisionCI": [
              0,
              0.434491
            ],
            "sample": "all"
          },
          "rejected": {
            "judged": 20,
            "of": 67,
            "real": 0,
            "precision": 0,
            "precisionCI": null,
            "sample": "uneven",
            "byRepository": [
              {
                "repository": "Openpanel-dev/openpanel",
                "of": 56,
                "judged": 14,
                "real": 0,
                "precisionCI": [
                  0,
                  0.215317
                ]
              },
              {
                "repository": "webstudio-is/webstudio",
                "of": 11,
                "judged": 6,
                "real": 0,
                "precisionCI": [
                  0,
                  0.390343
                ]
              }
            ]
          }
        },
        "realByRule": {},
        "calls": 134,
        "costUsd": 5.877113,
        "audit": [
          {
            "repository": "karakeep-app/karakeep",
            "entryPointsReviewed": 160,
            "problemsFound": 0,
            "coverage": "Reviewed the tRPC procedure bases in packages/trpc/index.ts (publicProcedure, authedProcedure, scoped and admin-scoped procedures) and listed every procedure in all 19 routers with its base. Traced payments fully: subscriptions router (checkout uses server-configured price ids chosen by an enum; syncWithStripe re-reads subscriptions from Stripe; admin-only manual tier grant) and the Hono /stripe webhook (raw c.req.text(), stripe.webhooks.constructEvent before writes, idempotent set of tier and quotas, deleted/updated events handled via a Stripe re-sync). Traced ownership middlewares (ensureBookmarkOwnership/Access, list viewer/editor/owner middlewares, highlight/feed/webhook/import-session services with Authorized<T> loaded per actor), Asset model (fromId checks canUserView; attach/replace/detach check asset and bookmark ownership), tag merge (tags filtered by user), list collaborator management (owner-only), user settings/avatar (explicit field pick, asset filtered by user), and the Hono REST routes for assets, public signed assets, rss and backups. Did not read every line of the 1600-line bookmarks router body (createBookmark, searchBookmarks, updateTags internals) or the workers app; public invite/verify/reset-password procedures were only checked for being token-based. No problem of the ten kinds verified."
          },
          {
            "repository": "papra-hq/papra",
            "entryPointsReviewed": 107,
            "problemsFound": 0,
            "coverage": "Enumerated all Hono routes in apps/papra-server/src/modules/**/*.routes.ts (107 handlers) and tabulated requireAuthentication / membership / admin guards. Traced payments: Stripe webhook reads context.req.text() and verifies with constructEventAsync before handleStripeWebhookEvent; subscription created/updated/deleted handled; checkout requires org owner and takes the price id from server plan config via a picklist planId; plan-entitlement claim checks driver eligibility with an external verification service. Traced organization-scoped routes (documents, tags, tagging rules, document views, custom properties, webhooks, intake emails, share links, documents batch, activity) down to repositories: reads/writes filter by organizationId or the record is first loaded with organizationId after ensureUserIsInOrganization. Checked member invite/role update/removal and invitation accept/reject/cancel/resend role and email checks, org restore (deletedBy check), API key delete (filtered by userId), PUT /api/users/me (name only), admin routes (requirePermissions), intake email ingest (HMAC signature), public share-link routes (token capability). Did not trace the tasks/cron workers or the client/mobile apps. No problem of the ten kinds verified."
          },
          {
            "repository": "stack-auth/stack",
            "entryPointsReviewed": 60,
            "problemsFound": 0,
            "coverage": "Of ~272 backend route files, reviewed: all 14 payments routes (create-purchase-url, validate-code, purchase-session, products list/grant/cancel/switch, invoices, billing, payment-method setup-intent/set-default, items read/update-quantity) plus the Stripe webhook (constructEvent on raw body buffer, claimStripeEvent dedup by event id, subscription deleted/updated events synced). Client-reachable payment routes use ensureClientCanAccessCustomer (self or team_admin, custom customers forbidden); inline products rejected for client; prices come from tenancy config; grant/update-quantity are server-only; switch updates the Stripe subscription with error_if_incomplete (test mode grant is a project setting). Reviewed the 11 user/team CRUD handlers with client schemas (sessions, connected accounts and access tokens, contact channels, notification preferences, oauth providers, project/team permissions, team invitations, team member profiles, team memberships, teams) and the shared crud definitions: every client operation checks auth.user against user_id or team membership/permission; client update schemas exclude server/read-only metadata and permission fields; permission and membership creation is server-only. Reviewed client-reachable non-CRUD routes (team invitation accept/send-code, contact-channel verification, internal platform-admin routes, internal conversations scoped by managed project, AI conversations by owner), neon/resend webhooks and the unsubscribe link (verification code). Did not review the remaining internal/admin-key routes (projects, api-keys, email templates, config, analytics, OAuth provider flows, auth sign-in/up internals) line by line. No problem of the ten kinds verified."
          }
        ]
      }
    ]
  },
  "screen": {
    "model": "ai-gateway:typesafe-ai/jev",
    "mode": "shadow",
    "rejectBelow": {
      "access-control": "shadow",
      "payment-integrity": "shadow",
      "identity-trust": "shadow",
      "ai-cost": "shadow",
      "ai-tool-abuse": "shadow",
      "login-rate-limit": "shadow",
      "state-changing-get": "shadow",
      "nosql-operator-injection": "shadow",
      "oauth-state-missing": "shadow",
      "webhook-signatures": "shadow",
      "regex-injection": "shadow",
      "webhook-verification-bypass": "shadow",
      "untrusted-jwt-key": "shadow",
      "prototype-pollution": "shadow",
      "archive-write-escape": "shadow"
    },
    "calibratedOn": {
      "date": "2026-09-26",
      "candidates": 616,
      "real": 555,
      "falseAlarms": 61
    }
  },
  "newer": {
    "scoredOn": "2026-09-23",
    "cases": 40,
    "perRuleBar": 0.75,
    "missedBar": [
      {
        "check": "identity-trust",
        "rule": "forgeable-identity",
        "found": 2,
        "of": 4
      }
    ]
  }
}