mirrors/guru
1
0
Fork 0
mirror of https://github.com/gentoo-mirror/guru.git synced 2026-07-16 18:43:18 -04:00
Code Issues Packages Projects Releases Wiki Activity

dev-python/tiktoken: new package, add 0.12.0

Browse Source 
Signed-off-by: Florian Albrechtskirchinger <falbrechtskirchinger@gmail.com>
This commit is contained in:
Florian Albrechtskirchinger
2026-03-26 03:00:34 +01:00
parent e59790c3e0
commit d999e1e10d
4 changed files with 166 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
dev-python/tiktoken/files/tiktoken-0.12.0-special-token-roudtrip.patch Normal file
View File

@@ -0,0 +1,13 @@
diff --git a/tests/test_encoding.py b/tests/test_encoding.py
index b77ca13..7ef65d9 100644
--- a/tests/test_encoding.py
+++ b/tests/test_encoding.py
@@ -152,7 +152,7 @@ def test_basic_roundtrip(make_enc):
def test_hyp_roundtrip(make_enc: Callable[[], tiktoken.Encoding], text):
enc = make_enc()
- assert text == enc.decode(enc.encode(text))
+ assert text == enc.decode(enc.encode(text, allowed_special="all"))
@pytest.mark.parametrize("make_enc", ENCODING_FACTORIES)