From cb9aa1b6ad7479ed8cbed2caad5f2ac089a66267 Mon Sep 17 00:00:00 2001 From: Florian Albrechtskirchinger Date: Wed, 8 Jul 2026 13:36:21 +0200 Subject: [PATCH] dev-python/tiktoken: add 0.13.0 Signed-off-by: Florian Albrechtskirchinger --- dev-python/tiktoken/Manifest | 7 ++ ...ktoken-0.13.0-special-token-roudtrip.patch | 13 +++ dev-python/tiktoken/tiktoken-0.13.0.ebuild | 92 +++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 dev-python/tiktoken/files/tiktoken-0.13.0-special-token-roudtrip.patch create mode 100644 dev-python/tiktoken/tiktoken-0.13.0.ebuild diff --git a/dev-python/tiktoken/Manifest b/dev-python/tiktoken/Manifest index 6ff16b4dca..79cdee3aea 100644 --- a/dev-python/tiktoken/Manifest +++ b/dev-python/tiktoken/Manifest @@ -13,10 +13,15 @@ DIST once_cell-1.21.4.crate 35010 BLAKE2B 9117ed1d478e626b7b7be2e3ff8b68a382b051 DIST portable-atomic-1.13.1.crate 197001 BLAKE2B 6d0898c2a537a9bf204b7749cca5f5fddf280772b56eb7ce8fdd9cf7d71561137b26ef89f415c40277a1c89981333eef78e5bb4624515a294ba28fdaeb4c70ea SHA512 2a1b31ac9814af884640b3398ab824a9795c72d260527a0966b193113808cfbb3345d50cd1beaebb45863437c3d06c8706d34b26efecceefc649319a4bc274a3 DIST proc-macro2-1.0.106.crate 59765 BLAKE2B 87f95795621c4c992c370dc3a1240a2d5580e3362356525efe640f8f26ad0492b3c88755d575c080ff7db11376b0d82f31141bf6f86c895249b85e46c081220f SHA512 b726e2c92af434bfa88cd4f53c3fe6db647503567675fb439890dee3d15f5111137e3242b28d164114ce081c10acf3fd11950753ddb349190c87ee04e7d97744 DIST pyo3-0.26.0.crate 1151579 BLAKE2B bcc8236785502928bf4a0b7a320e0a8f3d6140c562de4351a52d015aaebb8dd6b4d68fe607c8ea13ec66941c8f01d2c18c4249adfacb521355d4bf158667f3a3 SHA512 e073a4d893f88aa0301ef0ead78048137515edd93c490e6dccbc301ff65a208534c65c1d216b04639d4b1ec1e4b69bc87bd34974e5d0659134c28dcce589c194 +DIST pyo3-0.29.0.crate 1241253 BLAKE2B 29deb7fa70010bf3d7e9e93bbcd87c0b56c4ee7ef0c2e5956181d2fb5f06f4c9b3f28bd1ad280126c26c048808b91b847e1fb69ddd617ef9e9e5e004cf4e48ae SHA512 e6822400de53ebcef4a14c899dc065abae621a126483873139f5eb850877c94f4bccee3a183d4e2a69eadcd3c3b82b0a6f3e1053d926b103be572329607f2f67 DIST pyo3-build-config-0.26.0.crate 34309 BLAKE2B 5ea8b51da2f425674cd3bfcc1090ffdd08da01d2161dc66b08995c20cc859422f9f84077c545e5a0580182910aeb9dbba97838915148a82b86948d1cca3c8722 SHA512 a982d0a0baa4af58c2cef7c0c4b2671e3f559ca7f30586eda9fbd05e842efa2bfea0089ddbc729225b848871f6a25f85e86576e1dff519192ba289cd92c6aa8b +DIST pyo3-build-config-0.29.0.crate 39616 BLAKE2B 7b62a5976b890db84e5256f3136185c2a9c3b7110782b5b0cdc314a5231fde30a853ddcd17af63b53488b3e8d5e310b01002b6f3c32cfeb1e5c3e168cd6460d7 SHA512 df77e8d7d5fe173749437ff6b90fa1703083377d515ab8a85dc77d9cac9c3fc331ede4b040f7e3694311ddb5adff577cf90c181a27c196ab93a4569e918fdea8 DIST pyo3-ffi-0.26.0.crate 78247 BLAKE2B 65f1c399bc763bf8afc36ccac27d28242ebcccebd08ab8d9b3165ca5c9abb10f76e76fe5a250aa1e0cdc1d1e8a613160660f07bc0dbc2812675acb9dfc8e89c2 SHA512 91c2ae828e997dee5e48d839794727e94576228e9329c358a3bc77a59f8f37fa0ae6c2579229782b3f46ab90725844ce9194b9de4d3849d824adf1d40743bf7e +DIST pyo3-ffi-0.29.0.crate 88184 BLAKE2B 5891af92a3bcba764c4701dcc0a99edbced010a5ac46df3217d004f5c3a368f18331733c6723bd49e522c6f7105455df13e2be0f07695ed5ce6ae71c15dec438 SHA512 baaf01d13ddc09316ba16b22e9ca15a67e5bda5b861ed7961c3393fd4692fc1c96952f81465d1dcb6531b9083f96c5630981b3d7802cb1ac2af67aafaffbec76 DIST pyo3-macros-0.26.0.crate 8906 BLAKE2B 60eef72d27b5bc922b75f71ae24bb6f380288cb3a619bf2940b0de47b27e86cce0ffdb1fbdc9c0540d3f15c9df4884497aa1e6efec0f1579ac722b414818a428 SHA512 07bcddba0926c3dff7629da3260f9a38593fbb337b6a68d55c223f5944d912885e5319cd635ae1785026fed4adea1fcc19695ae83aca5bea127c5a8868fd2c43 +DIST pyo3-macros-0.29.0.crate 8915 BLAKE2B c12d23753d5aba7333183d4b41f16c8da1d4297a3ff86b3b150e9185c1978fc184ac198e566a41833292a8a78ab59446eafd5ef7169e5cb41493e7cbeaa68280 SHA512 fb6633ac76229d2e0ad88d51136a9318f03823b27c700a62cfbc957b4c834985e2f4c0cdaf5ea5e9993ec9e8ce3d63cd6b3fb823f2a042e65186adbed78da7c2 DIST pyo3-macros-backend-0.26.0.crate 81809 BLAKE2B 73885035855daf00b001e97e0a5ebe09412d212c8f01aa4db6638ef09990bf1bccbacf2e5a7e6ec21929bb1c2beba06587ecdb6ee954eda9e9c51c6c276cea95 SHA512 4146011a8ef18ecaa44b425eebe6e5e1370945063b98aaa83452e57b66aefbf5d483367b1d4d44aa5091303c4ba7f568c6428ecec0cfa6c90ba41631d4e99367 +DIST pyo3-macros-backend-0.29.0.crate 91943 BLAKE2B 7c2977dd9c80f3dd1f5375b0c65492b46048ca02e6355ad746bc8430cf92eaf25c701f416469eb69724fde5220651b1095c09eb9e3ff860176f862d7c7287407 SHA512 13a7af3958c09969637827d97ab9bbc664c407dda9e4283afaf495757798cf6f84932468bc092160fa75211ce0aafe6efc657f3a390234d77a67362cfb31fe0f DIST quote-1.0.45.crate 31564 BLAKE2B 1b2c2c3e6ebabfe10affa4c43ca0bfd1f52d8fd49e1df463e13e63a82de41272089c8e09e4c02e8a33cfee768ace26d4d42a50752de8fb5a776c59806139ad5d SHA512 df96ede01ebd0ed642b758bf1d0862d0e116fc2b3343e14edb657ce5438ccabe743dec4a0a40de4dee98c134d169af9aba459faadf9c7a832b26377e755d49b0 DIST regex-1.12.3.crate 153392 BLAKE2B 30f272fc2cda1e36219dfccf0338edacceb5bfada7ac81dd01429661b01b0fa72f6bb343e753e2388251aa0aadc0b5cf61e07670020c9ab351f418c23653ad47 SHA512 73ea9581c861490ad4c8c767c09f1f39be525f4298f5b19ba7b658dfade4c91f7ffd170214382ef4694855c0a48a4db57c39b75d8fa6a9312b07409170c66000 DIST regex-automata-0.4.14.crate 618012 BLAKE2B f12b0f1e59d32711c113f5f2157a4e1dbd05095c386f0e104ef3d551cf029d0c3346d09f54d5b8a8bdc557ab4d91ab80a224d81b3c522b3c64ad1566972c46cf SHA512 58f8e7c41f71c1df50fc2f4e70ea7f5ab762170494497541b9a09076058f94a799b6fc4da13c40fc26a2f1a0b650b499cb9830338152e96a24e3a32b4994a5e0 @@ -29,6 +34,8 @@ DIST serde_derive-1.0.228.crate 59605 BLAKE2B 166ee2f9ff9f7a0928d0a65a72db3d9226 DIST syn-2.0.117.crate 306011 BLAKE2B bbaec02c4322f522bdfc87caa4ce6fd1e09bbf8f4a102d9ed8728efacdd843a2750f616dce5b2d3a3e46bbb7d57439fbd10e328bc1d4a01e980986cd815798af SHA512 035c2e63542592b6a4058a2a82d4c2c1f5793f84fb1b04cbd393337a26fcf661a1f3bea439ce98a1fe943b1c4d36d2bda16df3bbc3fa6ca35e1548d530dc8416 DIST target-lexicon-0.13.5.crate 26928 BLAKE2B f8c62cb91f68a635e6ce3fc254df9ce378b5e8f62c1147404c967484b9ed97fb33d7993b4671611db76566063e999d658d201f856f70196ec678d72cc758cf6d SHA512 956df27c01e1186c5356117ffd07810fc0b62fb919d5bfd215955ba85be00ef675212a3c0dc36679986be7b825442fec5c93c7643b7e1e19c04572e6463546fc DIST tiktoken-0.12.0.tar.gz 37806 BLAKE2B 9cb14dd95205c347525dca92b64fc98e74a5ce8eb047346799bfddd99678c201c4bb4930687556bb0e03cab2d59e3626186391dec178da6b5574fcfa3c9c1161 SHA512 5439d8a109446303255bb3c4cfb51fb77a3137b9ea96b467665de11d48e26ef6eb774c9e3a2307dc870f332a67a8eb88966320cfff3ec0f6754081ff2e22831d +DIST tiktoken-0.13.0-crates.tar.xz 3133548 BLAKE2B a4958dee4d87af0869202ead56213c25c5b064b3aacd5e2ff11d2cf6ed812d00e6d46c71440e06eb73b684604c5941e99a0bbd85a5208dd0299712f9af0f2221 SHA512 fc06338034ab8de5f56b62ef32e2fe8843d86a4f02da04f8ee1e0db60b18f1bce61ca6ae944de224f81b7b8951528802af8543578a48683a9592d65ffad36fdb +DIST tiktoken-0.13.0.gh.tar.gz 35626 BLAKE2B 465489183c472e580cbb3f6ca91be2f70e7f3ff9b8ba980c89f1ceabae9056e295243e6ee35609a36c02e7ec7eb03525079aac598095c9544293c04dc55976c2 SHA512 93a1c8715bb656ea819ce73d1b2d4e4481bfdd6426379f81b366da3a868960b49255815c5a06aa9cb7ea7377f61bfa001c358051b923f509c44a1d989f66f2de DIST tiktoken-encodings-v2026.03.26.tar.xz 2274968 BLAKE2B 951249a33ea830f833a404fcc3adceed08374bbd191e2f9286b660b1d6649385a252431a1d63509c8e65a2493f5df4dc2bfce0770f326993497df51260df11ec SHA512 2678bc6c35c06adb187d15a7a627f3a9e19a8aea9b83241b2cafbd4ad117f287e4a9a8f82f831712bf12bb182d20e259af88f2da9b26d3be0f0a61fd35c5d7a5 DIST unicode-ident-1.0.24.crate 49298 BLAKE2B 34ba2906a3ebc06f52a4ba77dff96a8f3fb977718d260886fa9e051c5830765641b65a63977110f3a79f31fa177b385f81b01d56c80f6a0324d36eae72e2ebf0 SHA512 7be712d8d0075ac9341790180e312f66264d3e0255c91b38df0951baa5e446dec24551bb2125a66a1cb9b6ca2f1ea37c0cc15fa3dd74f398ecf4d5302c7dc259 DIST unindent-0.2.4.crate 7422 BLAKE2B 2e7870cd4f78240dbb1e5fb9c0f9b55c57bb40242fe668f105a0e862f1d6300e31efbffe0cdff676a5f96a8d19dfb148f88bfef1cd8a710556d777fd0f4ee37f SHA512 58bd4fd20a0b0a7200e0ea3ea70553cea135a5a8f7a2fb178520c41228f435becce4b9981019b7c73a17df87a2ee9b2a47c4cec29a3011bfe848d48b2473761b diff --git a/dev-python/tiktoken/files/tiktoken-0.13.0-special-token-roudtrip.patch b/dev-python/tiktoken/files/tiktoken-0.13.0-special-token-roudtrip.patch new file mode 100644 index 0000000000..b140e0d210 --- /dev/null +++ b/dev-python/tiktoken/files/tiktoken-0.13.0-special-token-roudtrip.patch @@ -0,0 +1,13 @@ +diff --git a/tests/test_encoding.py b/tests/test_encoding.py +index b77ca13..7ef65d9 100644 +--- a/tests/test_encoding.py ++++ b/tests/test_encoding.py +@@ -152,7 +152,7 @@ def test_basic_roundtrip(make_enc): + def test_hyp_roundtrip(make_enc: Callable[[], tiktoken.Encoding], text): + enc = make_enc() + +- assert text == enc.decode(enc.encode(text)) ++ assert text == enc.decode(enc.encode(text, disallowed_special=())) + + + @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) diff --git a/dev-python/tiktoken/tiktoken-0.13.0.ebuild b/dev-python/tiktoken/tiktoken-0.13.0.ebuild new file mode 100644 index 0000000000..693f2e4269 --- /dev/null +++ b/dev-python/tiktoken/tiktoken-0.13.0.ebuild @@ -0,0 +1,92 @@ +# Copyright 2026 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +DISTUTILS_EXT=1 +DISTUTILS_USE_PEP517=setuptools +PYTHON_COMPAT=( python3_{12..14} ) + +RUST_MIN_VER="1.85.0" +# Bump PyO3 to 0.29.0 (RUSTSEC-2026-0176, RUSTSEC-2026-0177) +CRATES=" + pyo3-build-config@0.29.0 + pyo3-ffi@0.29.0 + pyo3-macros-backend@0.29.0 + pyo3-macros@0.29.0 + pyo3@0.29.0 +" + +inherit cargo distutils-r1 optfeature + +# The encodings cache (tiktoken-encodings-*.tar.xz) holds files named after +# the SHA-1 of their URL. It can be generated from the source directory via: +# grep -Eo 'https://openaipublic.blob[^"]+' tiktoken_ext/openai_public.py | \ +# sort -u | while read u; do h=$(echo -n "$u" | sha1sum | awk '{print $1}'); \ +# wget -O "$h" "$u" ; done +# Include the license file from the source repo: +# https://github.com/openai/tiktoken/issues/92 +TTE_TAG=2026.03.26.0 +TTE_BASE_URI="https://github.com/falbrechtskirchinger/overlay-assets/releases/download" + +DESCRIPTION="A fast BPE tokeniser for use with OpenAI's models" +HOMEPAGE=" + https://github.com/openai/tiktoken + https://pypi.org/project/tiktoken/ +" +SRC_URI=" + https://github.com/openai/${PN}/archive/refs/tags/${PV}.tar.gz -> ${P}.gh.tar.gz + https://github.com/fasys-crate-dist/${PN}/releases/download/${PV}/${P}-crates.tar.xz + ${CARGO_CRATE_URIS} + test? ( + ${TTE_BASE_URI}/v${TTE_TAG}/tiktoken-encodings-v${TTE_TAG%.*}.tar.xz + ) +" + +LICENSE="MIT" +# Dependent crate licenses +LICENSE+=" Apache-2.0-with-LLVM-exceptions MIT Unicode-3.0" +SLOT="0" +KEYWORDS="~amd64 ~arm64" + +RDEPEND=" + dev-python/regex[${PYTHON_USEDEP}] + dev-python/requests[${PYTHON_USEDEP}] +" +BDEPEND=" + dev-python/setuptools-rust[${PYTHON_USEDEP}] + test? ( + dev-python/blobfile[${PYTHON_USEDEP}] + ) +" + +QA_FLAGS_IGNORED="usr/lib.*/py.*/site-packages/tiktoken/_tiktoken.*\.so" + +PATCHES=( + # test_encoding.py::test_hyp_roundtrip throws ValueError for special tokens + "${FILESDIR}/tiktoken-0.13.0-special-token-roudtrip.patch" +) + +EPYTEST_PLUGINS=( + hypothesis + pytest-{asyncio,timeout} +) +distutils_enable_tests pytest + +src_prepare() { + distutils-r1_src_prepare + sed '/^pyo3 =/s/version = "0\.28\.3"/version = "0.29.0"/' -i Cargo.toml || die +} + +python_test() { + local -x PATH=${BUILD_DIR}/install/usr/bin:${PATH} + local -x TIKTOKEN_CACHE_DIR="${WORKDIR}/tiktoken-encodings" + + rm -rf tiktoken || die + + epytest +} + +pkg_postinst() { + optfeature "reading GCS, ABS files" dev-python/blobfile +}