{
    "meta": {
        "complete": true,
        "started_at": "2025-02-01 13:39:56.467124Z",
        "commit": "96a9da6f8bdaa448951f8e4f8d0533b7d3719fe0",
        "platform": {
            "cargo_version": "cargo 1.82.0 (8f40fc59f 2024-08-21)",
            "rustc_version": "rustc 1.82.0 (f6e511eec 2024-10-15)",
            "python_version": "3.12.3",
            "platform": "Linux-6.8.0-1016-aws-x86_64-with-glibc2.39"
        },
        "ended_at": "2025-02-01 14:43:58.869373Z",
        "elapsed_ms": 3842402,
        "ragit_version": "ragit 0.3.0-dev",
        "commit_title": "add release note for 0.2.1",
        "commit_message": "",
        "rand_seed": 0
    },
    "tests": {
        "cargo_tests": {
            "seq": 0,
            "pass": false,
            "error": "\n#####################\n### path: command ###\n/home/ubuntu/Documents/ci/ragit/crates/ignore: cargo test\n\n### status_code ###\n101\n\n### stdout ###\n\nrunning 1 test\ntest tests::ignore_test ... FAILED\n\nfailures:\n\n---- tests::ignore_test stdout ----\nthread 'tests::ignore_test' panicked at src/tests.rs:90:9:\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"a/abc\") has to match, but but doesn't\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"abc\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"res/a.json\") has to match, but but doesn't\n(pattern: \"/*.json\" -> IgnorePattern { r: Regex(\"^/*\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n\n\nfailures:\n    tests::ignore_test\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s\n\n\n\n### stderr ###\n    Updating crates.io index\n     Locking 44 packages to latest compatible versions\n      Adding windows-core v0.52.0 (latest: v0.59.0)\n      Adding windows-targets v0.52.6 (latest: v0.53.0)\n      Adding windows_aarch64_gnullvm v0.52.6 (latest: v0.53.0)\n      Adding windows_aarch64_msvc v0.52.6 (latest: v0.53.0)\n      Adding windows_i686_gnu v0.52.6 (latest: v0.53.0)\n      Adding windows_i686_gnullvm v0.52.6 (latest: v0.53.0)\n      Adding windows_i686_msvc v0.52.6 (latest: v0.53.0)\n      Adding windows_x86_64_gnu v0.52.6 (latest: v0.53.0)\n      Adding windows_x86_64_gnullvm v0.52.6 (latest: v0.53.0)\n      Adding windows_x86_64_msvc v0.52.6 (latest: v0.53.0)\n   Compiling autocfg v1.4.0\n   Compiling memchr v2.7.4\n   Compiling num-traits v0.2.19\n   Compiling aho-corasick v1.1.3\n   Compiling iana-time-zone v0.1.61\n   Compiling regex-syntax v0.8.5\n   Compiling regex-automata v0.4.9\n   Compiling chrono v0.4.39\n   Compiling pathdiff v0.2.3\n   Compiling ragit-fs v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/fs)\n   Compiling regex v1.11.1\n   Compiling ragit-ignore v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/ignore)\n    Finished `test` profile [unoptimized + debuginfo] target(s) in 10.49s\n     Running unittests src/lib.rs (target/debug/deps/ragit_ignore-6a5412b4405563e3)\nerror: test failed, to rerun pass `--lib`\n\n\n\n\n#####################\n### path: command ###\n/home/ubuntu/Documents/ci/ragit/crates/ignore: cargo test --release\n\n### status_code ###\n101\n\n### stdout ###\n\nrunning 1 test\ntest tests::ignore_test ... FAILED\n\nfailures:\n\n---- tests::ignore_test stdout ----\nthread 'tests::ignore_test' panicked at src/tests.rs:90:9:\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"a/abc\") has to match, but but doesn't\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"abc\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"res/a.json\") has to match, but but doesn't\n(pattern: \"/*.json\" -> IgnorePattern { r: Regex(\"^/*\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n\n\nfailures:\n    tests::ignore_test\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\n\n\n\n### stderr ###\n   Compiling autocfg v1.4.0\n   Compiling memchr v2.7.4\n   Compiling num-traits v0.2.19\n   Compiling aho-corasick v1.1.3\n   Compiling regex-syntax v0.8.5\n   Compiling iana-time-zone v0.1.61\n   Compiling chrono v0.4.39\n   Compiling regex-automata v0.4.9\n   Compiling pathdiff v0.2.3\n   Compiling ragit-fs v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/fs)\n   Compiling regex v1.11.1\n   Compiling ragit-ignore v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/ignore)\n    Finished `release` profile [optimized] target(s) in 24.79s\n     Running unittests src/lib.rs (target/release/deps/ragit_ignore-ec76309b863e2fbc)\nerror: test failed, to rerun pass `--lib`\n\n\nTraceback (most recent call last):\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 332, in <module>\n    test()\n  File \"/home/ubuntu/Documents/ci/ragit/tests/cargo_tests.py\", line 45, in cargo_tests\n    raise Exception(\"\\n\\n\".join(errors))\nException: \n#####################\n### path: command ###\n/home/ubuntu/Documents/ci/ragit/crates/ignore: cargo test\n\n### status_code ###\n101\n\n### stdout ###\n\nrunning 1 test\ntest tests::ignore_test ... FAILED\n\nfailures:\n\n---- tests::ignore_test stdout ----\nthread 'tests::ignore_test' panicked at src/tests.rs:90:9:\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"a/abc\") has to match, but but doesn't\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"abc\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"res/a.json\") has to match, but but doesn't\n(pattern: \"/*.json\" -> IgnorePattern { r: Regex(\"^/*\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n\n\nfailures:\n    tests::ignore_test\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s\n\n\n\n### stderr ###\n    Updating crates.io index\n     Locking 44 packages to latest compatible versions\n      Adding windows-core v0.52.0 (latest: v0.59.0)\n      Adding windows-targets v0.52.6 (latest: v0.53.0)\n      Adding windows_aarch64_gnullvm v0.52.6 (latest: v0.53.0)\n      Adding windows_aarch64_msvc v0.52.6 (latest: v0.53.0)\n      Adding windows_i686_gnu v0.52.6 (latest: v0.53.0)\n      Adding windows_i686_gnullvm v0.52.6 (latest: v0.53.0)\n      Adding windows_i686_msvc v0.52.6 (latest: v0.53.0)\n      Adding windows_x86_64_gnu v0.52.6 (latest: v0.53.0)\n      Adding windows_x86_64_gnullvm v0.52.6 (latest: v0.53.0)\n      Adding windows_x86_64_msvc v0.52.6 (latest: v0.53.0)\n   Compiling autocfg v1.4.0\n   Compiling memchr v2.7.4\n   Compiling num-traits v0.2.19\n   Compiling aho-corasick v1.1.3\n   Compiling iana-time-zone v0.1.61\n   Compiling regex-syntax v0.8.5\n   Compiling regex-automata v0.4.9\n   Compiling chrono v0.4.39\n   Compiling pathdiff v0.2.3\n   Compiling ragit-fs v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/fs)\n   Compiling regex v1.11.1\n   Compiling ragit-ignore v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/ignore)\n    Finished `test` profile [unoptimized + debuginfo] target(s) in 10.49s\n     Running unittests src/lib.rs (target/debug/deps/ragit_ignore-6a5412b4405563e3)\nerror: test failed, to rerun pass `--lib`\n\n\n\n\n#####################\n### path: command ###\n/home/ubuntu/Documents/ci/ragit/crates/ignore: cargo test --release\n\n### status_code ###\n101\n\n### stdout ###\n\nrunning 1 test\ntest tests::ignore_test ... FAILED\n\nfailures:\n\n---- tests::ignore_test stdout ----\nthread 'tests::ignore_test' panicked at src/tests.rs:90:9:\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"a/abc\") has to match, but but doesn't\n(pattern: \"ab*\" -> IgnorePattern { r: Regex(\"^([^/]+/)*ab*$\") }, path: \"abc\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\n(pattern: \"*.json\" -> IgnorePattern { r: Regex(\"^([^/]+/)**\\\\.json$\") }, path: \"res/a.json\") has to match, but but doesn't\n(pattern: \"/*.json\" -> IgnorePattern { r: Regex(\"^/*\\\\.json$\") }, path: \"a.json\") has to match, but but doesn't\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n\n\nfailures:\n    tests::ignore_test\n\ntest result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s\n\n\n\n### stderr ###\n   Compiling autocfg v1.4.0\n   Compiling memchr v2.7.4\n   Compiling num-traits v0.2.19\n   Compiling aho-corasick v1.1.3\n   Compiling regex-syntax v0.8.5\n   Compiling iana-time-zone v0.1.61\n   Compiling chrono v0.4.39\n   Compiling regex-automata v0.4.9\n   Compiling pathdiff v0.2.3\n   Compiling ragit-fs v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/fs)\n   Compiling regex v1.11.1\n   Compiling ragit-ignore v0.3.0 (/home/ubuntu/Documents/ci/ragit/crates/ignore)\n    Finished `release` profile [optimized] target(s) in 24.79s\n     Running unittests src/lib.rs (target/release/deps/ragit_ignore-ec76309b863e2fbc)\nerror: test failed, to rerun pass `--lib`\n\n\n",
            "elapsed_ms": 1404195
        },
        "add_and_rm": {
            "seq": 1,
            "pass": true,
            "error": null,
            "elapsed_ms": 67009
        },
        "recover": {
            "seq": 2,
            "pass": true,
            "error": null,
            "elapsed_ms": 8426
        },
        "clone": {
            "seq": 3,
            "pass": false,
            "error": "Command '['cargo', 'run', '--release', '--', 'clone', 'http://127.0.0.1/test-user/repo1']' returned non-zero exit status 1.\nTraceback (most recent call last):\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 332, in <module>\n    test()\n  File \"/home/ubuntu/Documents/ci/ragit/tests/clone.py\", line 50, in clone\n    cargo_run([\"clone\", \"http://127.0.0.1/test-user/repo1\"])\n  File \"/home/ubuntu/Documents/ci/ragit/tests/utils.py\", line 61, in cargo_run\n    result = subprocess.run(args, **kwargs)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib/python3.12/subprocess.py\", line 571, in run\n    raise CalledProcessError(retcode, process.args,\nsubprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'clone', 'http://127.0.0.1/test-user/repo1']' returned non-zero exit status 1.\n",
            "elapsed_ms": 13516
        },
        "clone2": {
            "seq": 4,
            "pass": true,
            "error": null,
            "elapsed_ms": 381644
        },
        "cli": {
            "seq": 5,
            "pass": true,
            "error": null,
            "elapsed_ms": 6051
        },
        "many_chunks": {
            "seq": 6,
            "pass": true,
            "error": null,
            "elapsed_ms": 254990
        },
        "ls": {
            "seq": 7,
            "pass": true,
            "error": null,
            "elapsed_ms": 179294
        },
        "meta": {
            "seq": 8,
            "pass": true,
            "error": null,
            "elapsed_ms": 5481
        },
        "ii": {
            "seq": 9,
            "pass": false,
            "error": "tfidf result on term 'tfidf valu map term count' is not close enough. error: `answer[2] not in approximation`, answer: ['e8a5962f9b54ee9431c93df83595f5dd412edf0bc0b7f7e60000000100000216', '225c75fb823a51ff665653cd9b9c2267699bb2be227177dd0000000100000213', '929ddfe9cb7a78b787e784d556b588f8f1af57f829fa8aa40000000100000201', 'caa84862afaf775e3b6c14ade5ca017785e174f3c01cd7df0000000100000200', '1cb944f6e3c7a429669dfc1c444db1032ad9c89a018809590000000100000200', 'ef7dd93a7c14fc3a779d244c39da7482d2d412f62e4b5d060000000100000201', '1d864c20b618267b470944a67b2492269a17a9f3227b39590000000100000220', 'f13dfcfee4a0f4b756e9c743d22c2ca01dcacf7946e422320000000100000205', '29c35669031cf3a17a891913511593fcf23f78fbbaad181c0000000100000201', '05f4974fbc2aaec070abc9a84e7aa78028b56e4727585252000000010000021c'], approximation: ['e8a5962f9b54ee9431c93df83595f5dd412edf0bc0b7f7e60000000100000216', 'caa84862afaf775e3b6c14ade5ca017785e174f3c01cd7df0000000100000200', '225c75fb823a51ff665653cd9b9c2267699bb2be227177dd0000000100000213', 'ef7dd93a7c14fc3a779d244c39da7482d2d412f62e4b5d060000000100000201', '9e4bdf580cd9e559f9336df781c29edfb2d84655ad3451840000000100000203', '05f4974fbc2aaec070abc9a84e7aa78028b56e4727585252000000010000021c', 'f13dfcfee4a0f4b756e9c743d22c2ca01dcacf7946e422320000000100000205', '29c35669031cf3a17a891913511593fcf23f78fbbaad181c0000000100000201', '2a8c493194046ba39029062d15d0f6564a56c1256157481a0000000100000204', 'd5472e15d434698b801e4071d190e55c16e1a37ea7024dc900000001000001a0']\nTraceback (most recent call last):\n  File \"/home/ubuntu/Documents/ci/ragit/tests/ii.py\", line 88, in ii\n    raise AssertionError(f\"answer[{i}] not in approximation\")\nAssertionError: answer[2] not in approximation\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 332, in <module>\n    test()\n  File \"/home/ubuntu/Documents/ci/ragit/tests/ii.py\", line 104, in ii\n    raise AssertionError(f\"tfidf result on term '{term}' is not close enough. error: `{e}`, answer: {answer}, approximation: {approximation}\")\nAssertionError: tfidf result on term 'tfidf valu map term count' is not close enough. error: `answer[2] not in approximation`, answer: ['e8a5962f9b54ee9431c93df83595f5dd412edf0bc0b7f7e60000000100000216', '225c75fb823a51ff665653cd9b9c2267699bb2be227177dd0000000100000213', '929ddfe9cb7a78b787e784d556b588f8f1af57f829fa8aa40000000100000201', 'caa84862afaf775e3b6c14ade5ca017785e174f3c01cd7df0000000100000200', '1cb944f6e3c7a429669dfc1c444db1032ad9c89a018809590000000100000200', 'ef7dd93a7c14fc3a779d244c39da7482d2d412f62e4b5d060000000100000201', '1d864c20b618267b470944a67b2492269a17a9f3227b39590000000100000220', 'f13dfcfee4a0f4b756e9c743d22c2ca01dcacf7946e422320000000100000205', '29c35669031cf3a17a891913511593fcf23f78fbbaad181c0000000100000201', '05f4974fbc2aaec070abc9a84e7aa78028b56e4727585252000000010000021c'], approximation: ['e8a5962f9b54ee9431c93df83595f5dd412edf0bc0b7f7e60000000100000216', 'caa84862afaf775e3b6c14ade5ca017785e174f3c01cd7df0000000100000200', '225c75fb823a51ff665653cd9b9c2267699bb2be227177dd0000000100000213', 'ef7dd93a7c14fc3a779d244c39da7482d2d412f62e4b5d060000000100000201', '9e4bdf580cd9e559f9336df781c29edfb2d84655ad3451840000000100000203', '05f4974fbc2aaec070abc9a84e7aa78028b56e4727585252000000010000021c', 'f13dfcfee4a0f4b756e9c743d22c2ca01dcacf7946e422320000000100000205', '29c35669031cf3a17a891913511593fcf23f78fbbaad181c0000000100000201', '2a8c493194046ba39029062d15d0f6564a56c1256157481a0000000100000204', 'd5472e15d434698b801e4071d190e55c16e1a37ea7024dc900000001000001a0']\n",
            "elapsed_ms": 31070
        },
        "cat_file": {
            "seq": 10,
            "pass": true,
            "error": null,
            "elapsed_ms": 16906
        },
        "images": {
            "seq": 11,
            "pass": true,
            "error": null,
            "elapsed_ms": 4915
        },
        "markdown_reader": {
            "seq": 12,
            "pass": true,
            "error": null,
            "elapsed_ms": 9595
        },
        "csv_reader": {
            "seq": 13,
            "pass": true,
            "error": null,
            "elapsed_ms": 5470
        },
        "subdir": {
            "seq": 14,
            "pass": true,
            "error": null,
            "elapsed_ms": 15920
        },
        "tfidf": {
            "seq": 15,
            "pass": true,
            "error": null,
            "elapsed_ms": 18624
        },
        "merge": {
            "seq": 16,
            "pass": true,
            "error": null,
            "elapsed_ms": 35033
        },
        "external_bases": {
            "seq": 17,
            "pass": true,
            "error": null,
            "elapsed_ms": 65420
        },
        "end_to_end dummy": {
            "seq": 18,
            "pass": true,
            "error": null,
            "elapsed_ms": 56525
        },
        "end_to_end gpt-4o-mini": {
            "seq": 19,
            "pass": true,
            "error": null,
            "elapsed_ms": 140272
        },
        "prompts dummy": {
            "seq": 20,
            "pass": true,
            "error": null,
            "elapsed_ms": 8149
        },
        "prompts gpt-4o-mini": {
            "seq": 21,
            "pass": true,
            "error": null,
            "elapsed_ms": 73262
        },
        "prompts claude-3.5-sonnet": {
            "seq": 22,
            "pass": true,
            "error": null,
            "elapsed_ms": 90954
        },
        "empty dummy": {
            "seq": 23,
            "pass": true,
            "error": null,
            "elapsed_ms": 4134
        },
        "empty gpt-4o-mini": {
            "seq": 24,
            "pass": true,
            "error": null,
            "elapsed_ms": 5510
        },
        "images2 gpt-4o-mini": {
            "seq": 25,
            "pass": true,
            "error": null,
            "elapsed_ms": 15202
        },
        "images2 claude-3.5-sonnet": {
            "seq": 26,
            "pass": true,
            "error": null,
            "elapsed_ms": 18283
        },
        "ragit_api llama3.2-11b-groq": {
            "seq": 27,
            "pass": true,
            "error": null,
            "elapsed_ms": 8821
        },
        "ragit_api command-r": {
            "seq": 28,
            "pass": false,
            "error": "Command '['cargo', 'run', '--release', '--', '--model', 'command-r', '--input', 'hello_world.pdl']' returned non-zero exit status 101.\nTraceback (most recent call last):\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 332, in <module>\n    test()\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 302, in <lambda>\n    (\"ragit_api command-r\", lambda: ragit_api(test_model=\"command-r\")),\n                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/home/ubuntu/Documents/ci/ragit/tests/ragit_api.py\", line 9, in ragit_api\n    subprocess.run([\"cargo\", \"run\", \"--release\", '--', '--model', test_model, '--input', 'hello_world.pdl'], check=True)\n  File \"/usr/lib/python3.12/subprocess.py\", line 571, in run\n    raise CalledProcessError(retcode, process.args,\nsubprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', '--model', 'command-r', '--input', 'hello_world.pdl']' returned non-zero exit status 101.\n",
            "elapsed_ms": 414
        },
        "ragit_api phi-3-14b-ollama": {
            "seq": 29,
            "pass": false,
            "error": "Command '['cargo', 'run', '--release', '--', '--model', 'phi-3-14b-ollama', '--input', 'hello_world.pdl']' returned non-zero exit status 101.\nTraceback (most recent call last):\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 332, in <module>\n    test()\n  File \"/home/ubuntu/Documents/ci/ragit/tests/tests.py\", line 303, in <lambda>\n    (\"ragit_api phi-3-14b-ollama\", lambda: ragit_api(test_model=\"phi-3-14b-ollama\")),\n                                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/home/ubuntu/Documents/ci/ragit/tests/ragit_api.py\", line 9, in ragit_api\n    subprocess.run([\"cargo\", \"run\", \"--release\", '--', '--model', test_model, '--input', 'hello_world.pdl'], check=True)\n  File \"/usr/lib/python3.12/subprocess.py\", line 571, in run\n    raise CalledProcessError(retcode, process.args,\nsubprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', '--model', 'phi-3-14b-ollama', '--input', 'hello_world.pdl']' returned non-zero exit status 101.\n",
            "elapsed_ms": 317
        },
        "migrate": {
            "seq": 30,
            "pass": true,
            "error": null,
            "elapsed_ms": 494100
        }
    },
    "result": {
        "total": 31,
        "complete": 31,
        "pass": 26,
        "fail": 5,
        "remaining": 0
    }
}