total: 31, pass: 23, fail: 8
remove shots in some prompts
There are no more few-shots in `rerank_summary` and `rerank_title`. I'm
not sure how good it will get. I need more tests.
elapsed time: 1,418,183 ms
elapsed time: 69,460 ms
elapsed time: 8,589 ms
elapsed time: 13,659 ms
Command '['cargo', 'run', '--release', '--', 'clone', 'http://127.0.0.1/test-user/repo1']' returned non-zero exit status 1.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/clone.py", line 50, in clone
cargo_run(["clone", "http://127.0.0.1/test-user/repo1"])
File "/home/ubuntu/Documents/ci/ragit/tests/utils.py", line 61, in cargo_run
result = subprocess.run(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'clone', 'http://127.0.0.1/test-user/repo1']' returned non-zero exit status 1.
elapsed time: 381,017 ms
elapsed time: 7,329 ms
elapsed time: 277,924 ms
elapsed time: 180,004 ms
elapsed time: 5,418 ms
elapsed time: 45,763 ms
elapsed time: 18,082 ms
elapsed time: 6,081 ms
elapsed time: 10,998 ms
elapsed time: 5,933 ms
elapsed time: 16,875 ms
elapsed time: 19,715 ms
elapsed time: 35,471 ms
elapsed time: 68,906 ms
elapsed time: 23,420 ms
The build should have timed out
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 288, in <lambda>
("end_to_end dummy", lambda: end_to_end(test_model="dummy")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/end_to_end.py", line 78, in end_to_end
raise Exception("The build should have timed out")
Exception: The build should have timed out
elapsed time: 36,358 ms
Command '['cargo', 'run', '--release', '--', 'check', '--recover']' returned non-zero exit status 1.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 289, in <lambda>
("end_to_end gpt-4o-mini", lambda: end_to_end(test_model="gpt-4o-mini")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/end_to_end.py", line 80, in end_to_end
cargo_run(["check", "--recover"])
File "/home/ubuntu/Documents/ci/ragit/tests/utils.py", line 61, in cargo_run
result = subprocess.run(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'check', '--recover']' returned non-zero exit status 1.
elapsed time: 7,887 ms
Command '['cargo', 'run', '--release', '--', 'query', "You're looking at a source code of a command line utility. What does the main function do?"]' returned non-zero exit status 1.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 290, in <lambda>
("prompts dummy", lambda: prompts(test_model="dummy")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/prompts.py", line 52, in prompts
cargo_run(["query", "You're looking at a source code of a command line utility. What does the main function do?"])
File "/home/ubuntu/Documents/ci/ragit/tests/utils.py", line 61, in cargo_run
result = subprocess.run(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'query', "You're looking at a source code of a command line utility. What does the main function do?"]' returned non-zero exit status 1.
elapsed time: 57,938 ms
Command '['cargo', 'run', '--release', '--', 'query', "You're looking at a source code of a command line utility. What does the main function do?"]' returned non-zero exit status 1.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 291, in <lambda>
("prompts gpt-4o-mini", lambda: prompts(test_model="gpt-4o-mini")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/prompts.py", line 46, in prompts
cargo_run(["query", "You're looking at a source code of a command line utility. What does the main function do?"])
File "/home/ubuntu/Documents/ci/ragit/tests/utils.py", line 61, in cargo_run
result = subprocess.run(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'query', "You're looking at a source code of a command line utility. What does the main function do?"]' returned non-zero exit status 1.
elapsed time: 60,305 ms
Command '['cargo', 'run', '--release', '--', 'query', "You're looking at a source code of a command line utility. What does the main function do?"]' returned non-zero exit status 1.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 292, in <lambda>
("prompts claude-3.5-sonnet", lambda: prompts(test_model="claude-3.5-sonnet")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/prompts.py", line 46, in prompts
cargo_run(["query", "You're looking at a source code of a command line utility. What does the main function do?"])
File "/home/ubuntu/Documents/ci/ragit/tests/utils.py", line 61, in cargo_run
result = subprocess.run(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'query', "You're looking at a source code of a command line utility. What does the main function do?"]' returned non-zero exit status 1.
elapsed time: 4,296 ms
elapsed time: 5,281 ms
elapsed time: 12,855 ms
elapsed time: 12,780 ms
elapsed time: 9,177 ms
elapsed time: 414 ms
Command '['cargo', 'run', '--release', '--', '--model', 'command-r', '--input', 'hello_world.pdl']' returned non-zero exit status 101.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 302, in <lambda>
("ragit_api command-r", lambda: ragit_api(test_model="command-r")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/ragit_api.py", line 9, in ragit_api
subprocess.run(["cargo", "run", "--release", '--', '--model', test_model, '--input', 'hello_world.pdl'], check=True)
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', '--model', 'command-r', '--input', 'hello_world.pdl']' returned non-zero exit status 101.
elapsed time: 319 ms
Command '['cargo', 'run', '--release', '--', '--model', 'phi-3-14b-ollama', '--input', 'hello_world.pdl']' returned non-zero exit status 101.
Traceback (most recent call last):
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 332, in <module>
test()
File "/home/ubuntu/Documents/ci/ragit/tests/tests.py", line 303, in <lambda>
("ragit_api phi-3-14b-ollama", lambda: ragit_api(test_model="phi-3-14b-ollama")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/Documents/ci/ragit/tests/ragit_api.py", line 9, in ragit_api
subprocess.run(["cargo", "run", "--release", '--', '--model', test_model, '--input', 'hello_world.pdl'], check=True)
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', '--model', 'phi-3-14b-ollama', '--input', 'hello_world.pdl']' returned non-zero exit status 101.
elapsed time: 495,361 ms