{
  "configuration_ids": [
    "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c",
    "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9",
    "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572",
    "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f",
    "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8"
  ],
  "configurations": [
    {
      "configuration_id": "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c",
      "display": {
        "agent_adapter": "codex-json-fresh-home-bwrap",
        "agent_timeout_seconds": 300,
        "agent_version": "codex-cli 0.160.0",
        "effort": "medium",
        "isolation_profile": "linux-bwrap-fresh-home-v1",
        "kind": "codex",
        "label": "GPT-6 Luna via Codex CLI",
        "marker": "6L",
        "model": "gpt-6-luna",
        "network": "enabled",
        "series": "gpt6Luna"
      },
      "failed_task_ids": [
        "boot-hook-marker-survives-reboot",
        "devshell-tooling-contract",
        "dovecot-structured-settings-versions",
        "fleet-shared-profile-conflict",
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "lazy-type-error-boundary",
        "module-generated-jobs-fixed-point",
        "overlay-finalattrs-reoverride",
        "python-package-set-override-fleet",
        "rust-no-network-build",
        "stalwart-state-version-migration",
        "stdenv-cross-and-host-tools",
        "tor-onion-socket-outside-chroot",
        "vsftpd-local-login-without-pam",
        "wireless-eap-key-unreadable-after-hardening",
        "yggdrasil-credential-key-owner"
      ],
      "first_trial_at": "2026-10-04T14:59:32.432904+00:00",
      "incomplete_attempt_count": 0,
      "invalid_attempt_count": 0,
      "last_trial_at": "2026-10-04T16:00:23.246115+00:00",
      "mean_agent_seconds_per_task": 69.24063492063492,
      "mean_agent_seconds_per_trial": 1454.0533333333333,
      "observation_count": 63,
      "pass_at_1": 0.3174603174603175,
      "pass_at_k": {
        "1": 0.3174603174603175,
        "2": 0.41269841269841273,
        "3": 0.47619047619047616
      },
      "pass_hat_k": {
        "1": 0.31746031746031744,
        "2": 0.22222222222222224,
        "3": 0.19047619047619047
      },
      "pass_rate": {
        "estimate": 0.31746031746031744,
        "interval": {
          "high": 0.4400353174416858,
          "level": 0.95,
          "low": 0.21586686653519657
        },
        "n": 63,
        "successes": 20,
        "warnings": []
      },
      "run_ids": [
        "20261004T145808Z-a98123d1",
        "20261004T152711Z-e2afbbdb",
        "20261004T155626Z-6852c459"
      ],
      "study_ids": [
        "20261004T145807Z-6a2a2703"
      ],
      "task_count": 21,
      "tasks": {
        "boot-hook-marker-survives-reboot": {
          "failed_criteria": {
            "bastion-unchanged": 1,
            "ingestd-starts-first-boot": 2,
            "marker-after-reboot": 2,
            "prepare-once-per-boot": 1,
            "spool-on-volume": 2
          },
          "failure_classes": {
            "wrong-value": 2
          },
          "mean_agent_seconds": 85.60866666666668,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 75.7,
              "agent_timeout": false,
              "failed_criteria": [
                "bastion-unchanged",
                "ingestd-starts-first-boot",
                "marker-after-reboot",
                "prepare-once-per-boot",
                "spool-on-volume"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 100.0,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 81.126,
              "agent_timeout": false,
              "failed_criteria": [
                "ingestd-starts-first-boot",
                "marker-after-reboot",
                "spool-on-volume"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "devshell-tooling-contract": {
          "failed_criteria": {
            "experimental-features": 2,
            "formatter-contract": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 35.374,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 35.949,
              "agent_timeout": false,
              "failed_criteria": [
                "formatter-contract"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 28.982,
              "agent_timeout": false,
              "failed_criteria": [
                "experimental-features",
                "formatter-contract"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 41.191,
              "agent_timeout": false,
              "failed_criteria": [
                "experimental-features",
                "formatter-contract"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "dovecot-structured-settings-versions": {
          "failed_criteria": {
            "mail-storage-preserved": 3,
            "protocols-and-auth": 1,
            "tls-paths": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 132.87066666666666,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 155.034,
              "agent_timeout": false,
              "failed_criteria": [
                "mail-storage-preserved",
                "tls-paths"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 143.552,
              "agent_timeout": false,
              "failed_criteria": [
                "mail-storage-preserved",
                "protocols-and-auth",
                "tls-paths"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 100.026,
              "agent_timeout": false,
              "failed_criteria": [
                "mail-storage-preserved",
                "tls-paths"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "esphome-persistence-wrong-state-path": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 95.87400000000001,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 131.816,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 91.06,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 64.746,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "firewall-nftables-forwarding-silently-dropped": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 84.36333333333333,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 71.024,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 96.912,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 85.154,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "fleet-shared-profile-conflict": {
          "failed_criteria": {
            "follows-transport": 2,
            "hosts-evaluate": 2,
            "sendmail-unit": 3
          },
          "failure_classes": {
            "evaluation": 2,
            "overfit": 2,
            "wrong-value": 3
          },
          "mean_agent_seconds": 65.217,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 65.531,
              "agent_timeout": false,
              "failed_criteria": [
                "follows-transport",
                "hosts-evaluate",
                "sendmail-unit"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 85.672,
              "agent_timeout": false,
              "failed_criteria": [
                "sendmail-unit"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 44.448,
              "agent_timeout": false,
              "failed_criteria": [
                "follows-transport",
                "hosts-evaluate",
                "sendmail-unit"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "hardened-profile-import-no-op": {
          "failed_criteria": {
            "firewall-scoped": 1,
            "hosts-evaluate": 1,
            "kernel-hardening": 1,
            "network-hardening": 1
          },
          "failure_classes": {
            "evaluation": 1,
            "wrong-value": 1
          },
          "mean_agent_seconds": 127.81766666666665,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 147.354,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 80.369,
              "agent_timeout": false,
              "failed_criteria": [
                "firewall-scoped",
                "kernel-hardening",
                "network-hardening"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 155.73,
              "agent_timeout": false,
              "failed_criteria": [
                "hosts-evaluate"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "initrd-hook-systemd-stage1-migration": {
          "failed_criteria": {
            "root-on-mapper": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 60.645333333333326,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 80.749,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 42.611,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 58.576,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "lazy-type-error-boundary": {
          "failed_criteria": {
            "missing-fields-rejected": 3
          },
          "failure_classes": {
            "missing-attr": 3
          },
          "mean_agent_seconds": 31.655666666666665,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 37.626,
              "agent_timeout": false,
              "failed_criteria": [
                "missing-fields-rejected"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 27.418,
              "agent_timeout": false,
              "failed_criteria": [
                "missing-fields-rejected"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 29.923,
              "agent_timeout": false,
              "failed_criteria": [
                "missing-fields-rejected"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "module-generated-jobs-fixed-point": {
          "failed_criteria": {
            "coexists-with-other-modules": 3,
            "disabled-jobs": 3,
            "option-jobs-evaluate": 3,
            "unit-fields": 3,
            "unit-names": 3
          },
          "failure_classes": {
            "evaluation": 3,
            "wrong-value": 3
          },
          "mean_agent_seconds": 54.730999999999995,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 43.855,
              "agent_timeout": false,
              "failed_criteria": [
                "coexists-with-other-modules",
                "disabled-jobs",
                "option-jobs-evaluate",
                "unit-fields",
                "unit-names"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 60.444,
              "agent_timeout": false,
              "failed_criteria": [
                "coexists-with-other-modules",
                "disabled-jobs",
                "option-jobs-evaluate",
                "unit-fields",
                "unit-names"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 59.894,
              "agent_timeout": false,
              "failed_criteria": [
                "coexists-with-other-modules",
                "disabled-jobs",
                "option-jobs-evaluate",
                "unit-fields",
                "unit-names"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "overlay-finalattrs-reoverride": {
          "failed_criteria": {
            "dependency-reoverride": 2
          },
          "failure_classes": {
            "wrong-value": 2
          },
          "mean_agent_seconds": 49.50633333333334,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 27.906,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 73.384,
              "agent_timeout": false,
              "failed_criteria": [
                "dependency-reoverride"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 47.229,
              "agent_timeout": false,
              "failed_criteria": [
                "dependency-reoverride"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "python-package-set-override-fleet": {
          "failed_criteria": {
            "apps-instantiate": 3
          },
          "failure_classes": {
            "evaluation": 3
          },
          "mean_agent_seconds": 74.34433333333332,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 73.588,
              "agent_timeout": false,
              "failed_criteria": [
                "apps-instantiate"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 103.327,
              "agent_timeout": false,
              "failed_criteria": [
                "apps-instantiate"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 46.118,
              "agent_timeout": false,
              "failed_criteria": [
                "apps-instantiate"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "rust-no-network-build": {
          "failed_criteria": {
            "system-onnxruntime": 1
          },
          "failure_classes": {
            "impurity": 1
          },
          "mean_agent_seconds": 35.88166666666667,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 31.055,
              "agent_timeout": false,
              "failed_criteria": [
                "system-onnxruntime"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 30.689,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 45.901,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "service-requires-removed-network-setup": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 54.25366666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 52.118,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 51.475,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 59.168,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "stalwart-state-version-migration": {
          "failed_criteria": {
            "secrets-and-backup-follow": 3,
            "unit-overrides-applied": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 70.38833333333334,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 59.656,
              "agent_timeout": false,
              "failed_criteria": [
                "secrets-and-backup-follow",
                "unit-overrides-applied"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 79.08,
              "agent_timeout": false,
              "failed_criteria": [
                "secrets-and-backup-follow",
                "unit-overrides-applied"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 72.429,
              "agent_timeout": false,
              "failed_criteria": [
                "secrets-and-backup-follow",
                "unit-overrides-applied"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "state-directory-ownership-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 65.962,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 68.624,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 50.078,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 79.184,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "stdenv-cross-and-host-tools": {
          "failed_criteria": {
            "generator-build-compiler": 2,
            "instantiates": 2,
            "man-page-from-scdoc": 3,
            "pkg-config-and-libraries": 2,
            "wrapper-host-zstd": 2
          },
          "failure_classes": {
            "evaluation": 2,
            "wrong-value": 3
          },
          "mean_agent_seconds": 50.230333333333334,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 58.469,
              "agent_timeout": false,
              "failed_criteria": [
                "generator-build-compiler",
                "instantiates",
                "man-page-from-scdoc",
                "pkg-config-and-libraries",
                "wrapper-host-zstd"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 38.235,
              "agent_timeout": false,
              "failed_criteria": [
                "generator-build-compiler",
                "instantiates",
                "man-page-from-scdoc",
                "pkg-config-and-libraries",
                "wrapper-host-zstd"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 53.987,
              "agent_timeout": false,
              "failed_criteria": [
                "man-page-from-scdoc"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "tor-onion-socket-outside-chroot": {
          "failed_criteria": {
            "socket-dir-lifecycle": 3,
            "tor-after-backend": 2,
            "tor-binds-socket-dir": 2
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 77.06433333333334,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 44.997,
              "agent_timeout": false,
              "failed_criteria": [
                "socket-dir-lifecycle",
                "tor-after-backend",
                "tor-binds-socket-dir"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 90.403,
              "agent_timeout": false,
              "failed_criteria": [
                "socket-dir-lifecycle"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 95.793,
              "agent_timeout": false,
              "failed_criteria": [
                "socket-dir-lifecycle",
                "tor-after-backend",
                "tor-binds-socket-dir"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "vsftpd-local-login-without-pam": {
          "failed_criteria": {
            "mirror-unchanged": 1
          },
          "failure_classes": {
            "wrong-value": 1
          },
          "mean_agent_seconds": 88.52133333333332,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 71.761,
              "agent_timeout": false,
              "failed_criteria": [
                "mirror-unchanged"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 67.193,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 126.61,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "wireless-eap-key-unreadable-after-hardening": {
          "failed_criteria": {
            "daemon-hardened": 2,
            "eap-config": 2,
            "hosts-evaluate": 1,
            "key-provisioned": 1,
            "psk-host-unchanged": 1
          },
          "failure_classes": {
            "evaluation": 1,
            "wrong-value": 2
          },
          "mean_agent_seconds": 66.11933333333333,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 60.822,
              "agent_timeout": false,
              "failed_criteria": [
                "daemon-hardened",
                "eap-config",
                "hosts-evaluate",
                "key-provisioned",
                "psk-host-unchanged"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 74.12,
              "agent_timeout": false,
              "failed_criteria": [
                "daemon-hardened",
                "eap-config"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 63.416,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        },
        "yggdrasil-credential-key-owner": {
          "failed_criteria": {
            "edge-evaluates": 3,
            "edge-key-from-runtime-secret": 3,
            "key-never-in-store": 3
          },
          "failure_classes": {
            "evaluation": 3,
            "impurity": 3,
            "wrong-value": 3
          },
          "mean_agent_seconds": 47.62433333333333,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 51.522,
              "agent_timeout": false,
              "failed_criteria": [
                "edge-evaluates",
                "edge-key-from-runtime-secret",
                "key-never-in-store"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-a98123d1"
            },
            {
              "agent_duration_seconds": 39.342,
              "agent_timeout": false,
              "failed_criteria": [
                "edge-evaluates",
                "edge-key-from-runtime-secret",
                "key-never-in-store"
              ],
              "passed": false,
              "run_id": "20261004T152711Z-e2afbbdb"
            },
            {
              "agent_duration_seconds": 52.009,
              "agent_timeout": false,
              "failed_criteria": [
                "edge-evaluates",
                "edge-key-from-runtime-secret",
                "key-never-in-store"
              ],
              "passed": false,
              "run_id": "20261004T155626Z-6852c459"
            }
          ]
        }
      },
      "timeout_count": 0,
      "timeout_rate": 0.0,
      "trial_count": 3,
      "unsolved_task_ids": [
        "devshell-tooling-contract",
        "dovecot-structured-settings-versions",
        "fleet-shared-profile-conflict",
        "initrd-hook-systemd-stage1-migration",
        "lazy-type-error-boundary",
        "module-generated-jobs-fixed-point",
        "python-package-set-override-fleet",
        "stalwart-state-version-migration",
        "stdenv-cross-and-host-tools",
        "tor-onion-socket-outside-chroot",
        "yggdrasil-credential-key-owner"
      ]
    },
    {
      "configuration_id": "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9",
      "display": {
        "agent_adapter": "codex-json-fresh-home-bwrap",
        "agent_timeout_seconds": 300,
        "agent_version": "codex-cli 0.160.0",
        "effort": "medium",
        "isolation_profile": "linux-bwrap-fresh-home-v1",
        "kind": "codex",
        "label": "GPT-6 Astra via Codex CLI",
        "marker": "6A",
        "model": "gpt-6-astra",
        "network": "enabled",
        "series": "gpt6Astra"
      },
      "failed_task_ids": [
        "devshell-tooling-contract",
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "rust-no-network-build",
        "tor-onion-socket-outside-chroot"
      ],
      "first_trial_at": "2026-10-04T15:01:15.714464+00:00",
      "incomplete_attempt_count": 0,
      "invalid_attempt_count": 0,
      "last_trial_at": "2026-10-04T16:20:27.461272+00:00",
      "mean_agent_seconds_per_task": 100.26884126984127,
      "mean_agent_seconds_per_trial": 2105.645666666667,
      "observation_count": 63,
      "pass_at_1": 0.8253968253968254,
      "pass_at_k": {
        "1": 0.8253968253968254,
        "2": 0.8571428571428571,
        "3": 0.8571428571428571
      },
      "pass_hat_k": {
        "1": 0.8253968253968254,
        "2": 0.7936507936507937,
        "3": 0.7619047619047619
      },
      "pass_rate": {
        "estimate": 0.8253968253968254,
        "interval": {
          "high": 0.8996060342053211,
          "level": 0.95,
          "low": 0.7137857247924102
        },
        "n": 63,
        "successes": 52,
        "warnings": []
      },
      "run_ids": [
        "20261004T145808Z-1c5b9b61",
        "20261004T153841Z-d363bfec",
        "20261004T161734Z-824a5bc7"
      ],
      "study_ids": [
        "20261004T145807Z-c450ef1c"
      ],
      "task_count": 21,
      "tasks": {
        "boot-hook-marker-survives-reboot": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 110.641,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 106.226,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 128.376,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 97.321,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "devshell-tooling-contract": {
          "failed_criteria": {
            "experimental-features": 1
          },
          "failure_classes": {
            "wrong-value": 1
          },
          "mean_agent_seconds": 56.464,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 78.432,
              "agent_timeout": false,
              "failed_criteria": [
                "experimental-features"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 48.928,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 42.032,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "dovecot-structured-settings-versions": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 118.49466666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 143.681,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 116.355,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 95.448,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "esphome-persistence-wrong-state-path": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 99.801,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 112.535,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 88.416,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 98.452,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "firewall-nftables-forwarding-silently-dropped": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 111.524,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 104.923,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 83.709,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 145.94,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "fleet-shared-profile-conflict": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 104.98033333333332,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 94.978,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 116.106,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 103.857,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "hardened-profile-import-no-op": {
          "failed_criteria": {
            "kernel-hardening": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 188.64466666666667,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 123.943,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 149.169,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 292.822,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "initrd-hook-systemd-stage1-migration": {
          "failed_criteria": {
            "root-on-mapper": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 122.502,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 90.382,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 140.887,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 136.237,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "lazy-type-error-boundary": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 78.22866666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 89.508,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 83.778,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 61.4,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "module-generated-jobs-fixed-point": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 71.15933333333334,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 64.699,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 71.267,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 77.512,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "overlay-finalattrs-reoverride": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 87.355,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 91.478,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 81.67,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 88.917,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "python-package-set-override-fleet": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 99.79366666666665,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 103.172,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 83.63,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 112.579,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "rust-no-network-build": {
          "failed_criteria": {
            "pinned-model": 1
          },
          "failure_classes": {
            "impurity": 1
          },
          "mean_agent_seconds": 65.00233333333334,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 66.83,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 68.648,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 59.529,
              "agent_timeout": false,
              "failed_criteria": [
                "pinned-model"
              ],
              "passed": false,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "service-requires-removed-network-setup": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 124.89966666666668,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 133.727,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 107.378,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 133.594,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "stalwart-state-version-migration": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 88.31966666666666,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 75.34,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 102.483,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 87.136,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "state-directory-ownership-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 99.521,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 103.261,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 90.149,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 105.153,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "stdenv-cross-and-host-tools": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 56.11033333333334,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 62.942,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 62.365,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 43.024,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "tor-onion-socket-outside-chroot": {
          "failed_criteria": {
            "tor-after-backend": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 129.091,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 104.167,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 98.97,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 184.136,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "vsftpd-local-login-without-pam": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 91.91566666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 106.154,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 83.927,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 85.666,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "wireless-eap-key-unreadable-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 116.238,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 130.446,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 103.717,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 114.551,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        },
        "yggdrasil-credential-key-owner": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 84.95966666666668,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 79.273,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-1c5b9b61"
            },
            {
              "agent_duration_seconds": 79.29,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153841Z-d363bfec"
            },
            {
              "agent_duration_seconds": 96.316,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T161734Z-824a5bc7"
            }
          ]
        }
      },
      "timeout_count": 0,
      "timeout_rate": 0.0,
      "trial_count": 3,
      "unsolved_task_ids": [
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "tor-onion-socket-outside-chroot"
      ]
    },
    {
      "configuration_id": "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572",
      "display": {
        "agent_adapter": "claude-json-bwrap",
        "agent_timeout_seconds": 300,
        "agent_version": "2.1.288 (Claude Code)",
        "effort": "default",
        "isolation_profile": "linux-bwrap-fresh-home-v1",
        "kind": "claude",
        "label": "Claude Sonnet 5.5 via Claude Code",
        "marker": "S5",
        "model": "claude-sonnet-5-5",
        "network": "enabled",
        "series": "claudeSonnet55"
      },
      "failed_task_ids": [
        "boot-hook-marker-survives-reboot",
        "esphome-persistence-wrong-state-path",
        "fleet-shared-profile-conflict",
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "lazy-type-error-boundary",
        "tor-onion-socket-outside-chroot",
        "wireless-eap-key-unreadable-after-hardening"
      ],
      "first_trial_at": "2026-10-04T14:59:53.861566+00:00",
      "incomplete_attempt_count": 0,
      "invalid_attempt_count": 0,
      "last_trial_at": "2026-10-04T15:35:33.913079+00:00",
      "mean_agent_seconds_per_task": 34.29457142857143,
      "mean_agent_seconds_per_trial": 720.186,
      "observation_count": 63,
      "pass_at_1": 0.746031746031746,
      "pass_at_k": {
        "1": 0.746031746031746,
        "2": 0.8412698412698413,
        "3": 0.9047619047619048
      },
      "pass_hat_k": {
        "1": 0.746031746031746,
        "2": 0.6507936507936508,
        "3": 0.6190476190476191
      },
      "pass_rate": {
        "estimate": 0.746031746031746,
        "interval": {
          "high": 0.8371958872667884,
          "level": 0.95,
          "low": 0.6265881256339354
        },
        "n": 63,
        "successes": 47,
        "warnings": []
      },
      "run_ids": [
        "20261004T145808Z-35aa8694",
        "20261004T151616Z-9b9eb3f0",
        "20261004T153214Z-e12ea0a7"
      ],
      "study_ids": [
        "20261004T145807Z-6afd814c"
      ],
      "task_count": 21,
      "tasks": {
        "boot-hook-marker-survives-reboot": {
          "failed_criteria": {
            "ingestd-starts-first-boot": 1,
            "marker-after-reboot": 1,
            "prepare-once-per-boot": 1
          },
          "failure_classes": {
            "wrong-value": 1
          },
          "mean_agent_seconds": 32.311,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 20.786,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 29.085,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 47.062,
              "agent_timeout": false,
              "failed_criteria": [
                "ingestd-starts-first-boot",
                "marker-after-reboot",
                "prepare-once-per-boot"
              ],
              "passed": false,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "devshell-tooling-contract": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 11.81,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 11.993,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 13.255,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 10.182,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "dovecot-structured-settings-versions": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 77.341,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 53.879,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 57.628,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 120.516,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "esphome-persistence-wrong-state-path": {
          "failed_criteria": {
            "other-persistence-unchanged": 2
          },
          "failure_classes": {
            "wrong-value": 2
          },
          "mean_agent_seconds": 30.909666666666666,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 29.674,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 30.327,
              "agent_timeout": false,
              "failed_criteria": [
                "other-persistence-unchanged"
              ],
              "passed": false,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 32.728,
              "agent_timeout": false,
              "failed_criteria": [
                "other-persistence-unchanged"
              ],
              "passed": false,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "firewall-nftables-forwarding-silently-dropped": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 29.784666666666666,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 25.342,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 34.289,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 29.723,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "fleet-shared-profile-conflict": {
          "failed_criteria": {
            "baseline-kept": 1,
            "sendmail-unit": 1
          },
          "failure_classes": {
            "wrong-value": 2
          },
          "mean_agent_seconds": 46.369,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 42.003,
              "agent_timeout": false,
              "failed_criteria": [
                "sendmail-unit"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 43.111,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 53.993,
              "agent_timeout": false,
              "failed_criteria": [
                "baseline-kept"
              ],
              "passed": false,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "hardened-profile-import-no-op": {
          "failed_criteria": {
            "hosts-evaluate": 1,
            "kernel-hardening": 2
          },
          "failure_classes": {
            "evaluation": 1,
            "wrong-value": 2
          },
          "mean_agent_seconds": 49.925000000000004,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 42.074,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 42.651,
              "agent_timeout": false,
              "failed_criteria": [
                "hosts-evaluate",
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 65.05,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "initrd-hook-systemd-stage1-migration": {
          "failed_criteria": {
            "root-on-mapper": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 53.26233333333334,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 62.025,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 47.154,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 50.608,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "lazy-type-error-boundary": {
          "failed_criteria": {
            "missing-fields-rejected": 1
          },
          "failure_classes": {
            "missing-attr": 1
          },
          "mean_agent_seconds": 15.371,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 14.442,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 14.69,
              "agent_timeout": false,
              "failed_criteria": [
                "missing-fields-rejected"
              ],
              "passed": false,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 16.981,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "module-generated-jobs-fixed-point": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 19.397333333333332,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 17.97,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 20.55,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 19.672,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "overlay-finalattrs-reoverride": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 16.619666666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 14.9,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 16.891,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 18.068,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "python-package-set-override-fleet": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 49.511,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 59.64,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 44.296,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 44.597,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "rust-no-network-build": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 17.102,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 15.621,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 18.859,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 16.826,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "service-requires-removed-network-setup": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 37.594,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 27.086,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 46.859,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 38.837,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "stalwart-state-version-migration": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 30.667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 27.835,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 30.829,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 33.337,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "state-directory-ownership-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 28.131666666666664,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 25.274,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 29.22,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 29.901,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "stdenv-cross-and-host-tools": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 26.028000000000002,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 27.441,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 27.416,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 23.227,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "tor-onion-socket-outside-chroot": {
          "failed_criteria": {
            "socket-dir-lifecycle": 1,
            "tor-after-backend": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 41.085,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 43.706,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 37.546,
              "agent_timeout": false,
              "failed_criteria": [
                "socket-dir-lifecycle",
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 42.003,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "vsftpd-local-login-without-pam": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 34.81133333333333,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 35.766,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 34.082,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 34.586,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "wireless-eap-key-unreadable-after-hardening": {
          "failed_criteria": {
            "eap-config": 2
          },
          "failure_classes": {
            "wrong-value": 2
          },
          "mean_agent_seconds": 36.25033333333334,
          "n": 3,
          "passes": 1,
          "solve_rate": 0.3333333333333333,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 45.286,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 28.486,
              "agent_timeout": false,
              "failed_criteria": [
                "eap-config"
              ],
              "passed": false,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 34.979,
              "agent_timeout": false,
              "failed_criteria": [
                "eap-config"
              ],
              "passed": false,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        },
        "yggdrasil-credential-key-owner": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 35.905,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 37.586,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-35aa8694"
            },
            {
              "agent_duration_seconds": 33.246,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T151616Z-9b9eb3f0"
            },
            {
              "agent_duration_seconds": 36.883,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T153214Z-e12ea0a7"
            }
          ]
        }
      },
      "timeout_count": 0,
      "timeout_rate": 0.0,
      "trial_count": 3,
      "unsolved_task_ids": [
        "initrd-hook-systemd-stage1-migration",
        "tor-onion-socket-outside-chroot"
      ]
    },
    {
      "configuration_id": "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f",
      "display": {
        "agent_adapter": "codex-json-fresh-home-bwrap",
        "agent_timeout_seconds": 300,
        "agent_version": "codex-cli 0.160.0",
        "effort": "medium",
        "isolation_profile": "linux-bwrap-fresh-home-v1",
        "kind": "codex",
        "label": "GPT-6.1 Sol via Codex CLI",
        "marker": "6S",
        "model": "gpt-6.1-sol",
        "network": "enabled",
        "series": "gpt61Sol"
      },
      "failed_task_ids": [
        "devshell-tooling-contract",
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "rust-no-network-build",
        "tor-onion-socket-outside-chroot"
      ],
      "first_trial_at": "2026-10-04T15:02:15.891300+00:00",
      "incomplete_attempt_count": 0,
      "invalid_attempt_count": 0,
      "last_trial_at": "2026-10-04T16:39:57.149011+00:00",
      "mean_agent_seconds_per_task": 121.4391111111111,
      "mean_agent_seconds_per_trial": 2550.2213333333334,
      "observation_count": 63,
      "pass_at_1": 0.8253968253968254,
      "pass_at_k": {
        "1": 0.8253968253968254,
        "2": 0.8571428571428571,
        "3": 0.8571428571428571
      },
      "pass_hat_k": {
        "1": 0.8253968253968254,
        "2": 0.7936507936507937,
        "3": 0.7619047619047619
      },
      "pass_rate": {
        "estimate": 0.8253968253968254,
        "interval": {
          "high": 0.8996060342053211,
          "level": 0.95,
          "low": 0.7137857247924102
        },
        "n": 63,
        "successes": 52,
        "warnings": []
      },
      "run_ids": [
        "20261004T145808Z-ccdf993a",
        "20261004T154851Z-44e9c91f",
        "20261004T163812Z-2e7ace0c"
      ],
      "study_ids": [
        "20261004T145808Z-5f75e815"
      ],
      "task_count": 21,
      "tasks": {
        "boot-hook-marker-survives-reboot": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 141.29666666666665,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 167.004,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 158.305,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 98.581,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "devshell-tooling-contract": {
          "failed_criteria": {
            "experimental-features": 1
          },
          "failure_classes": {
            "wrong-value": 1
          },
          "mean_agent_seconds": 74.77833333333332,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 79.712,
              "agent_timeout": false,
              "failed_criteria": [
                "experimental-features"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 84.582,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 60.041,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "dovecot-structured-settings-versions": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 143.83866666666665,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 124.089,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 142.537,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 164.89,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "esphome-persistence-wrong-state-path": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 119.03666666666668,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 118.959,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 129.621,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 108.53,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "firewall-nftables-forwarding-silently-dropped": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 164.816,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 160.678,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 154.153,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 179.617,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "fleet-shared-profile-conflict": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 168.00699999999998,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 147.546,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 225.636,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 130.839,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "hardened-profile-import-no-op": {
          "failed_criteria": {
            "kernel-hardening": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 138.76133333333334,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 144.717,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 150.548,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 121.019,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "initrd-hook-systemd-stage1-migration": {
          "failed_criteria": {
            "root-on-mapper": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 156.61366666666666,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 166.951,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 156.946,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 145.944,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "lazy-type-error-boundary": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 80.045,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 85.766,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 70.001,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 84.368,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "module-generated-jobs-fixed-point": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 89.55366666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 83.144,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 104.214,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 81.303,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "overlay-finalattrs-reoverride": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 101.71766666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 85.597,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 86.234,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 133.322,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "python-package-set-override-fleet": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 151.438,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 160.958,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 156.392,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 136.964,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "rust-no-network-build": {
          "failed_criteria": {
            "pinned-model": 1
          },
          "failure_classes": {
            "impurity": 1
          },
          "mean_agent_seconds": 64.32366666666667,
          "n": 3,
          "passes": 2,
          "solve_rate": 0.6666666666666666,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 59.65,
              "agent_timeout": false,
              "failed_criteria": [
                "pinned-model"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 66.963,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 66.358,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "service-requires-removed-network-setup": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 150.38466666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 178.301,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 143.098,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 129.755,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "stalwart-state-version-migration": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 99.85866666666668,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 104.477,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 104.183,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 90.916,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "state-directory-ownership-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 125.54533333333335,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 128.94,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 148.175,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 99.521,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "stdenv-cross-and-host-tools": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 82.45100000000001,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 79.363,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 104.332,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 63.658,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "tor-onion-socket-outside-chroot": {
          "failed_criteria": {
            "tor-after-backend": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 143.83133333333333,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 204.549,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 89.145,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 137.8,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "vsftpd-local-login-without-pam": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 116.08466666666668,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 107.511,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 122.261,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 118.482,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "wireless-eap-key-unreadable-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 143.43933333333334,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 164.952,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 137.761,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 127.605,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        },
        "yggdrasil-credential-key-owner": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 94.39999999999999,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 91.722,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ccdf993a"
            },
            {
              "agent_duration_seconds": 101.874,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T154851Z-44e9c91f"
            },
            {
              "agent_duration_seconds": 89.604,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T163812Z-2e7ace0c"
            }
          ]
        }
      },
      "timeout_count": 0,
      "timeout_rate": 0.0,
      "trial_count": 3,
      "unsolved_task_ids": [
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "tor-onion-socket-outside-chroot"
      ]
    },
    {
      "configuration_id": "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8",
      "display": {
        "agent_adapter": "claude-json-bwrap",
        "agent_timeout_seconds": 300,
        "agent_version": "2.1.288 (Claude Code)",
        "effort": "default",
        "isolation_profile": "linux-bwrap-fresh-home-v1",
        "kind": "claude",
        "label": "Claude Opus 5.5 via Claude Code",
        "marker": "O5",
        "model": "claude-opus-5-5",
        "network": "enabled",
        "series": "claudeOpus55"
      },
      "failed_task_ids": [
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "service-requires-removed-network-setup",
        "tor-onion-socket-outside-chroot"
      ],
      "first_trial_at": "2026-10-04T15:00:40.889032+00:00",
      "incomplete_attempt_count": 0,
      "invalid_attempt_count": 0,
      "last_trial_at": "2026-10-04T15:54:22.124121+00:00",
      "mean_agent_seconds_per_task": 60.964333333333336,
      "mean_agent_seconds_per_trial": 1280.251,
      "observation_count": 63,
      "pass_at_1": 0.8095238095238095,
      "pass_at_k": {
        "1": 0.8095238095238095,
        "2": 0.8095238095238095,
        "3": 0.8095238095238095
      },
      "pass_hat_k": {
        "1": 0.8095238095238095,
        "2": 0.8095238095238095,
        "3": 0.8095238095238095
      },
      "pass_rate": {
        "estimate": 0.8095238095238095,
        "interval": {
          "high": 0.8875380812770755,
          "level": 0.95,
          "low": 0.6959321285012545
        },
        "n": 63,
        "successes": 51,
        "warnings": []
      },
      "run_ids": [
        "20261004T145808Z-ac180016",
        "20261004T152637Z-36f0dd12",
        "20261004T155204Z-c7a68bb9"
      ],
      "study_ids": [
        "20261004T145808Z-54bfd030"
      ],
      "task_count": 21,
      "tasks": {
        "boot-hook-marker-survives-reboot": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 57.885333333333335,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 70.157,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 46.912,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 56.587,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "devshell-tooling-contract": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 21.706666666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 19.456,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 25.457,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 20.207,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "dovecot-structured-settings-versions": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 72.867,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 72.722,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 68.472,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 77.407,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "esphome-persistence-wrong-state-path": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 82.69833333333334,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 82.563,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 81.767,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 83.765,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "firewall-nftables-forwarding-silently-dropped": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 62.01466666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 70.393,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 52.051,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 63.6,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "fleet-shared-profile-conflict": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 93.53733333333334,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 109.662,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 92.276,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 78.674,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "hardened-profile-import-no-op": {
          "failed_criteria": {
            "kernel-hardening": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 114.706,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 120.852,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 110.933,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 112.333,
              "agent_timeout": false,
              "failed_criteria": [
                "kernel-hardening"
              ],
              "passed": false,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "initrd-hook-systemd-stage1-migration": {
          "failed_criteria": {
            "root-on-mapper": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 104.45933333333333,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 110.742,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 115.81,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 86.826,
              "agent_timeout": false,
              "failed_criteria": [
                "root-on-mapper"
              ],
              "passed": false,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "lazy-type-error-boundary": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 27.913666666666668,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 30.942,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 26.877,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 25.922,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "module-generated-jobs-fixed-point": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 23.246333333333336,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 20.933,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 26.09,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 22.716,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "overlay-finalattrs-reoverride": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 29.464,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 29.939,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 26.574,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 31.879,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "python-package-set-override-fleet": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 60.80166666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 62.331,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 52.881,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 67.193,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "rust-no-network-build": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 22.117333333333335,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 21.832,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 21.71,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 22.81,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "service-requires-removed-network-setup": {
          "failed_criteria": {
            "vendor-password-login": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 78.66833333333334,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 77.852,
              "agent_timeout": false,
              "failed_criteria": [
                "vendor-password-login"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 65.443,
              "agent_timeout": false,
              "failed_criteria": [
                "vendor-password-login"
              ],
              "passed": false,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 92.71,
              "agent_timeout": false,
              "failed_criteria": [
                "vendor-password-login"
              ],
              "passed": false,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "stalwart-state-version-migration": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 51.44366666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 50.823,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 52.377,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 51.131,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "state-directory-ownership-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 51.73333333333333,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 46.919,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 60.466,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 47.815,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "stdenv-cross-and-host-tools": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 45.66166666666667,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 48.191,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 47.102,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 41.692,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "tor-onion-socket-outside-chroot": {
          "failed_criteria": {
            "tor-after-backend": 3
          },
          "failure_classes": {
            "wrong-value": 3
          },
          "mean_agent_seconds": 80.962,
          "n": 3,
          "passes": 0,
          "solve_rate": 0.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 85.715,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 79.616,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 77.555,
              "agent_timeout": false,
              "failed_criteria": [
                "tor-after-backend"
              ],
              "passed": false,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "vsftpd-local-login-without-pam": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 55.79433333333333,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 66.902,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 51.745,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 48.736,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "wireless-eap-key-unreadable-after-hardening": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 80.86399999999999,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 80.332,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 82.482,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 79.778,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        },
        "yggdrasil-credential-key-owner": {
          "failed_criteria": {},
          "failure_classes": {},
          "mean_agent_seconds": 61.705999999999996,
          "n": 3,
          "passes": 3,
          "solve_rate": 1.0,
          "timeouts": 0,
          "trials": [
            {
              "agent_duration_seconds": 67.365,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T145808Z-ac180016"
            },
            {
              "agent_duration_seconds": 51.133,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T152637Z-36f0dd12"
            },
            {
              "agent_duration_seconds": 66.62,
              "agent_timeout": false,
              "failed_criteria": [],
              "passed": true,
              "run_id": "20261004T155204Z-c7a68bb9"
            }
          ]
        }
      },
      "timeout_count": 0,
      "timeout_rate": 0.0,
      "trial_count": 3,
      "unsolved_task_ids": [
        "hardened-profile-import-no-op",
        "initrd-hook-systemd-stage1-migration",
        "service-requires-removed-network-setup",
        "tor-onion-socket-outside-chroot"
      ]
    }
  ],
  "corpus": {
    "digest": "f783a51cb0ab545778cf74f03dbe97c4e2a6025b0b7ed293eed6e35c224ba380",
    "id": "nixbench-public",
    "version": "3.2.0"
  },
  "generated_at": "2026-10-04T18:55:01.361185+00:00",
  "methods": {
    "discrimination": "point-biserial-leave-one-task-out",
    "discrimination_thresholds": {
      "minimum_configurations": 2,
      "minimum_observations": 20
    },
    "headline": "task-pass-rate",
    "pass_at_k": "unbiased estimator 1 - C(n-c,k)/C(n,k), averaged over tasks",
    "pass_hat_k": "C(c,k)/C(n,k), averaged over tasks (all k sampled trials pass)",
    "pass_rate_interval": "wilson-95 over pooled valid task-trial observations; cells of the same task are not independent, so the interval is descriptive",
    "saturation_pooled_solve_rate": 0.9,
    "solve_rate_bands": {
      "high": ">= 0.75",
      "low": "< 0.25",
      "mixed": "0.25 to < 0.75"
    }
  },
  "saturated_task_ids": [
    "firewall-nftables-forwarding-silently-dropped",
    "state-directory-ownership-after-hardening",
    "vsftpd-local-login-without-pam"
  ],
  "schema_version": 1,
  "task_ids": [
    "boot-hook-marker-survives-reboot",
    "devshell-tooling-contract",
    "dovecot-structured-settings-versions",
    "esphome-persistence-wrong-state-path",
    "firewall-nftables-forwarding-silently-dropped",
    "fleet-shared-profile-conflict",
    "hardened-profile-import-no-op",
    "initrd-hook-systemd-stage1-migration",
    "lazy-type-error-boundary",
    "module-generated-jobs-fixed-point",
    "overlay-finalattrs-reoverride",
    "python-package-set-override-fleet",
    "rust-no-network-build",
    "service-requires-removed-network-setup",
    "stalwart-state-version-migration",
    "state-directory-ownership-after-hardening",
    "stdenv-cross-and-host-tools",
    "tor-onion-socket-outside-chroot",
    "vsftpd-local-login-without-pam",
    "wireless-eap-key-unreadable-after-hardening",
    "yggdrasil-credential-key-owner"
  ],
  "tasks": {
    "boot-hook-marker-survives-reboot": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.3333333333333333,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.6666666666666666,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "devshell-tooling-contract": {
      "author_difficulty": "easy",
      "category": "devshells",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "mixed",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.6666666666666666,
        "successes": 10
      },
      "solve_rate_band": "mixed",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 0.6666666666666666,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 0.6666666666666666,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "dovecot-structured-settings-versions": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "esphome-persistence-wrong-state-path": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8666666666666667,
        "successes": 13
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 1.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.3333333333333333,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "firewall-nftables-forwarding-silently-dropped": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "saturated",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 1.0,
        "successes": 15
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 1.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "fleet-shared-profile-conflict": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "mixed",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.6666666666666666,
        "successes": 10
      },
      "solve_rate_band": "mixed",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.3333333333333333,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "hardened-profile-import-no-op": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "hard",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.13333333333333333,
        "successes": 2
      },
      "solve_rate_band": "low",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.3333333333333333,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 0.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.3333333333333333,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 0.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 0.0
      }
    },
    "initrd-hook-systemd-stage1-migration": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "hard",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.0,
        "successes": 0
      },
      "solve_rate_band": "low",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 0.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 0.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 0.0
      }
    },
    "lazy-type-error-boundary": {
      "author_difficulty": "hard",
      "category": "nix-language",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "mixed",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.7333333333333333,
        "successes": 11
      },
      "solve_rate_band": "mixed",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.6666666666666666,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "module-generated-jobs-fixed-point": {
      "author_difficulty": "medium",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "overlay-finalattrs-reoverride": {
      "author_difficulty": "hard",
      "category": "overlays",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8666666666666667,
        "successes": 13
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.3333333333333333,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "python-package-set-override-fleet": {
      "author_difficulty": "hard",
      "category": "overlays",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "rust-no-network-build": {
      "author_difficulty": "hard",
      "category": "packages",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.6666666666666666,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 0.6666666666666666,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 0.6666666666666666,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "service-requires-removed-network-setup": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 1.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 0.0
      }
    },
    "stalwart-state-version-migration": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "state-directory-ownership-after-hardening": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "saturated",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 1.0,
        "successes": 15
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 1.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "stdenv-cross-and-host-tools": {
      "author_difficulty": "hard",
      "category": "packages",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "tor-onion-socket-outside-chroot": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "hard",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.0,
        "successes": 0
      },
      "solve_rate_band": "low",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 0.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 0.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 0.0
      }
    },
    "vsftpd-local-login-without-pam": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "saturated",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.9333333333333333,
        "successes": 14
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.6666666666666666,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "wireless-eap-key-unreadable-after-hardening": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "mixed",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.7333333333333333,
        "successes": 11
      },
      "solve_rate_band": "mixed",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.3333333333333333,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 0.3333333333333333,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    },
    "yggdrasil-credential-key-owner": {
      "author_difficulty": "hard",
      "category": "modules",
      "discrimination": {
        "configuration_count": 5,
        "estimate": null,
        "leave_one_task_out_statistic": "sum-normalized-task-scores",
        "method": "point-biserial-leave-one-task-out",
        "method_version": "1",
        "n": 15,
        "sampling_unit": "valid-task-observation",
        "unavailable_reason": "insufficient-sample-size"
      },
      "empirical_difficulty": "easy",
      "pooled": {
        "configuration_count": 5,
        "n": 15,
        "solve_rate": 0.8,
        "successes": 12
      },
      "solve_rate_band": "high",
      "solve_rate_by_configuration": {
        "cfg-2d6c0b618a9dd0bd17ae590a7131c283d43b9b208d83ffbf29515f24ad824b0c": 0.0,
        "cfg-358ffc419a241aabb9b64378c2f6b500693726b4224cb15061831864c954d3f9": 1.0,
        "cfg-54941733dac6c69c011212ae525bf62eec5ba9f011019a4f185e8e1d29ba3572": 1.0,
        "cfg-770e1a4eb3d04a7bd2bb801a5696bfa20a85bf24442ed45b0fe57d5ae01e5a6f": 1.0,
        "cfg-cebeadcb604ae5026cbba1aef72a18a4e259cc48cf089934b0526415932485e8": 1.0
      }
    }
  }
}