total: 57, pass: 51, fail: 6
fix plain text reader
Now it rejects to process text files that are not utf-8. It's because
1. Now that ragit continues processing files even if there's an
erroneous file, it's okay to throw more errors. It'll not bother the
users.
2. Plain text reader is the default file reader. If a user mistakenly
adds a random file, which is likely to be a binary file, ragit will
use the plain text reader. If it's using `String::from_utf8_lossy`,
it'll generate a chunk with tons of REPLACEMENT_CHARACTERs, which is
total waste of time and energy.
elapsed time: 486,116 ms
elapsed time: 139,760 ms
elapsed time: 46,391 ms
elapsed time: 14,760 ms
elapsed time: 12,524 ms
elapsed time: 136,480 ms
elapsed time: 10,888 ms
elapsed time: 17,191 ms
elapsed time: 9,773 ms
{'ragit_version': '0.4.0-dev', 'chunk_count': 0, 'staged_files': [], 'processed_files': {}, 'curr_processing_file': None, 'repo_url': None, 'ii_status': {'type': 'None'}, 'uid': None} != {'ragit_version': '0.4.0-dev', 'chunk_count': 0, 'staged_files': [], 'processed_files': {}, 'curr_processing_file': None, 'repo_url': None, 'ii_status': {'type': 'None'}, 'uid': {'high': 226965926617079404232257257017206136310, 'low': 98353210059702837338669213515912314881}}
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 630, in <module>
test()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/server.py", line 51, in server
assert_eq_json("index.json", index_json)
File "/Users/baehyunsol/Documents/Rust/ragit/tests/server.py", line 266, in assert_eq_json
raise ValueError(f"{file.__repr__()} != {value.__repr__()}")
ValueError: {'ragit_version': '0.4.0-dev', 'chunk_count': 0, 'staged_files': [], 'processed_files': {}, 'curr_processing_file': None, 'repo_url': None, 'ii_status': {'type': 'None'}, 'uid': None} != {'ragit_version': '0.4.0-dev', 'chunk_count': 0, 'staged_files': [], 'processed_files': {}, 'curr_processing_file': None, 'repo_url': None, 'ii_status': {'type': 'None'}, 'uid': {'high': 226965926617079404232257257017206136310, 'low': 98353210059702837338669213515912314881}}
elapsed time: 2,529 ms
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 630, in <module>
test()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/server_permission.py", line 23, in server_permission
create_user(id="test-user-2", email="sample2@email.com", password="abcdefgh")
File "/Users/baehyunsol/Documents/Rust/ragit/tests/server.py", line 186, in create_user
assert response.status_code == 200
^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
elapsed time: 25,360 ms
elapsed time: 430,526 ms
elapsed time: 449,215 ms
elapsed time: 61,263 ms
elapsed time: 310,972 ms
elapsed time: 8,005 ms
elapsed time: 10,789 ms
elapsed time: 137,816 ms
tfidf result on term 'search gpg annot select correspond' is not close enough. error: `approximation[2] not in answer`, answer: ['3e0d93ece16c10490435c08b7b755db9a57e53b818a9e62c0000000100000fa3', 'c5719c769542cb0cde49558784948082703f2da9618c29d80000000100000fb3', '6d1b2eeef26e5ce9672e62a7ca43412c66b86ad0e48d27620000000100000fa0', '606389435f969a017ad1cf63a7a30eba0d1a08c743efea9f0000000100000318', 'f386d96798aad5baf548b6985b367932bdc89483b756b515000000010000081f', 'c66345d5ab119b4cf05a6899472b54a4fd0041ee2b83b9f80000000100000fa2', 'bf8735875031f53ccd50e48e6674d9ac64c90f68bb0c7edb0000000100000fa0', '509b4b369f9f9729365a6947ce43335209d934562feeb7220000000100000fa2', '82ad9747a31109a3ef965e4168a0968cb56a448390416e290000000100000bf5', 'b632241f25a98c9320097079669e1acd10afd534e67ec2600000000100000fa2'], approximation: ['3e0d93ece16c10490435c08b7b755db9a57e53b818a9e62c0000000100000fa3', 'b632241f25a98c9320097079669e1acd10afd534e67ec2600000000100000fa2', '90a25e1efdafffab6369490140eecabb90ab0649108feeff0000000100000cd4', 'bf8735875031f53ccd50e48e6674d9ac64c90f68bb0c7edb0000000100000fa0', '5cdbfe828a4a84a4129bda3cc32bb8376914275561fa6a1a0000000100000da8', 'c5719c769542cb0cde49558784948082703f2da9618c29d80000000100000fb3', '0833e100c47da17ca6a2d202310483ed3c08f75ec2cfbf4a0000000100000c67', '1ff3d753fa4b857385f748c5d02a7371332241a8579211f9000000010000075c', '6f305111c4ab2bb2243ce34889afb4f72dff498303da56890000000100000c1e', '6d1b2eeef26e5ce9672e62a7ca43412c66b86ad0e48d27620000000100000fa0']
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/ii.py", line 103, in ii_worker
raise AssertionError(f"approximation[{i}] not in answer")
AssertionError: approximation[2] not in answer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 630, in <module>
test()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/ii.py", line 49, in ii
ii_worker()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/ii.py", line 116, in ii_worker
raise AssertionError(f"tfidf result on term '{term}' is not close enough. error: `{e}`, answer: {answer}, approximation: {approximation}")
AssertionError: tfidf result on term 'search gpg annot select correspond' is not close enough. error: `approximation[2] not in answer`, answer: ['3e0d93ece16c10490435c08b7b755db9a57e53b818a9e62c0000000100000fa3', 'c5719c769542cb0cde49558784948082703f2da9618c29d80000000100000fb3', '6d1b2eeef26e5ce9672e62a7ca43412c66b86ad0e48d27620000000100000fa0', '606389435f969a017ad1cf63a7a30eba0d1a08c743efea9f0000000100000318', 'f386d96798aad5baf548b6985b367932bdc89483b756b515000000010000081f', 'c66345d5ab119b4cf05a6899472b54a4fd0041ee2b83b9f80000000100000fa2', 'bf8735875031f53ccd50e48e6674d9ac64c90f68bb0c7edb0000000100000fa0', '509b4b369f9f9729365a6947ce43335209d934562feeb7220000000100000fa2', '82ad9747a31109a3ef965e4168a0968cb56a448390416e290000000100000bf5', 'b632241f25a98c9320097079669e1acd10afd534e67ec2600000000100000fa2'], approximation: ['3e0d93ece16c10490435c08b7b755db9a57e53b818a9e62c0000000100000fa3', 'b632241f25a98c9320097079669e1acd10afd534e67ec2600000000100000fa2', '90a25e1efdafffab6369490140eecabb90ab0649108feeff0000000100000cd4', 'bf8735875031f53ccd50e48e6674d9ac64c90f68bb0c7edb0000000100000fa0', '5cdbfe828a4a84a4129bda3cc32bb8376914275561fa6a1a0000000100000da8', 'c5719c769542cb0cde49558784948082703f2da9618c29d80000000100000fb3', '0833e100c47da17ca6a2d202310483ed3c08f75ec2cfbf4a0000000100000c67', '1ff3d753fa4b857385f748c5d02a7371332241a8579211f9000000010000075c', '6f305111c4ab2bb2243ce34889afb4f72dff498303da56890000000100000c1e', '6d1b2eeef26e5ce9672e62a7ca43412c66b86ad0e48d27620000000100000fa0']
elapsed time: 37,604 ms
elapsed time: 32,073 ms
elapsed time: 10,952 ms
elapsed time: 15,309 ms
elapsed time: 9,131 ms
elapsed time: 311,819 ms
Command '['cargo', 'run', '--release', '--', 'build']' returned non-zero exit status 1.
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 630, in <module>
test()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/real_repos.py", line 63, in real_repos
cargo_run(["build"])
File "/Users/baehyunsol/Documents/Rust/ragit/tests/utils.py", line 70, in cargo_run
result = subprocess.run(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/anaconda3/lib/python3.11/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['cargo', 'run', '--release', '--', 'build']' returned non-zero exit status 1.
elapsed time: 28,669 ms
elapsed time: 32,065 ms
elapsed time: 52,132 ms
elapsed time: 52,524 ms
elapsed time: 81,439 ms
elapsed time: 99,604 ms
elapsed time: 15,851 ms
elapsed time: 7,773 ms
elapsed time: 9,675 ms
elapsed time: 49,523 ms
elapsed time: 74,632 ms
elapsed time: 10,979 ms
elapsed time: 11,634 ms
elapsed time: 16,215 ms
Expecting value: line 1 column 1 (char 0)
Traceback (most recent call last):
File "/opt/anaconda3/lib/python3.11/site-packages/requests/models.py", line 974, in json
return complexjson.loads(self.text, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/anaconda3/lib/python3.11/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/anaconda3/lib/python3.11/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/anaconda3/lib/python3.11/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 630, in <module>
test()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 578, in <lambda>
("server_chat llama3.3-70b", lambda: server_chat(test_model="llama3.3-70b")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/baehyunsol/Documents/Rust/ragit/tests/server_chat.py", line 67, in server_chat
responses2.append(requests.post(f"http://127.0.0.1:41127/test-user/sample2/chat/{chat_id2}", files={"query": "How does the rust compiler implement type system?"}).json())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/anaconda3/lib/python3.11/site-packages/requests/models.py", line 978, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
elapsed time: 13,375 ms
elapsed time: 12,642 ms
elapsed time: 8,073 ms
elapsed time: 145,263 ms
elapsed time: 18,883 ms
Traceback (most recent call last):
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 630, in <module>
test()
File "/Users/baehyunsol/Documents/Rust/ragit/tests/tests.py", line 583, in <lambda>
("svg gpt-4o-mini", lambda: svg(test_model="gpt-4o-mini")),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/baehyunsol/Documents/Rust/ragit/tests/svg.py", line 123, in svg
assert "ragit" in cargo_run(["pdl", "test1.pdl"], stdout=True).lower()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
elapsed time: 40,598 ms
elapsed time: 15,367 ms
elapsed time: 3,174 ms
elapsed time: 10,871 ms
elapsed time: 116,872 ms
elapsed time: 84,096 ms
elapsed time: 191,770 ms
elapsed time: 9,394 ms
elapsed time: 3,683 ms
elapsed time: 975 ms
elapsed time: 466 ms
elapsed time: 260,554 ms
elapsed time: 69,247 ms
elapsed time: 37,666 ms