diff --git a/recipes/curated-tokenizers/patches/0001-use-our-own-sentencepiece-abseil.patch b/recipes/curated-tokenizers/patches/0001-use-our-own-sentencepiece-abseil.patch index df4ee220a6e08..0a089291c9e9b 100644 --- a/recipes/curated-tokenizers/patches/0001-use-our-own-sentencepiece-abseil.patch +++ b/recipes/curated-tokenizers/patches/0001-use-our-own-sentencepiece-abseil.patch @@ -1,12 +1,12 @@ -From cc968a797278f4862db877a9449fadcc60f455b7 Mon Sep 17 00:00:00 2001 +From 3fd938029f2aa350d9a401b1380975f8f767745f Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 25 Nov 2024 08:07:00 +1100 Subject: [PATCH] use our own sentencepiece/abseil --- curated_tokenizers/_spp.pxd | 2 +- - setup.py | 38 +++++++++++++++++++++++++------------ - 2 files changed, 27 insertions(+), 13 deletions(-) + setup.py | 41 ++++++++++++++++++++++++++----------- + 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/curated_tokenizers/_spp.pxd b/curated_tokenizers/_spp.pxd index 29f5ec2..4630b45 100644 @@ -22,7 +22,7 @@ index 29f5ec2..4630b45 100644 uint32_t id() const const string & piece() const diff --git a/setup.py b/setup.py -index d0aa11b..0306041 100755 +index d0aa11b..efa7ac1 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,6 @@ @@ -53,11 +53,12 @@ index d0aa11b..0306041 100755 "-pthread", "-DHAVE_PTHREAD=1", ], -@@ -139,21 +140,34 @@ def setup_package(): +@@ -139,21 +140,37 @@ def setup_package(): if len(sys.argv) > 1 and sys.argv[1] == "clean": return clean(root / "curated_tokenizers") + library_path = pathlib.Path(os.environ['PREFIX']) ++ extra_link_args = [] + EXTENSION_LIBRARIES = ["sentencepiece"] + if "win32" in sys.platform: + absl_libs = ( @@ -67,6 +68,7 @@ index d0aa11b..0306041 100755 + 'absl_flags_usage', 'absl_flags_usage_internal', + ) + EXTENSION_LIBRARIES += ["libprotobuf"] ++ extra_link_args + ["/NODEFAULTLIB:libprotobuf.lib"] + library_path = library_path / "Library" + else: + absl_glob = pathlib.Path(os.environ['PREFIX']).glob('lib/libabsl_*.so') @@ -94,6 +96,7 @@ index d0aa11b..0306041 100755 ], + libraries=EXTENSION_LIBRARIES, + library_dirs=[str(library_path)], ++ extra_link_args=extra_link_args, language="c++", ), Extension(