Skip to content

Commit

Permalink
use /NODEFAULLIB for libprotobuf
Browse files Browse the repository at this point in the history
unfortunately, sentencepiece is still static,
and we get duplicate symbols otherwise
  • Loading branch information
h-vetinari committed Nov 25, 2024
1 parent aa2910c commit 5d63c4f
Showing 1 changed file with 8 additions and 5 deletions.
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
From cc968a797278f4862db877a9449fadcc60f455b7 Mon Sep 17 00:00:00 2001
From 3fd938029f2aa350d9a401b1380975f8f767745f Mon Sep 17 00:00:00 2001
From: "H. Vetinari" <h.vetinari@gmx.com>
Date: Mon, 25 Nov 2024 08:07:00 +1100
Subject: [PATCH] use our own sentencepiece/abseil

---
curated_tokenizers/_spp.pxd | 2 +-
setup.py | 38 +++++++++++++++++++++++++------------
2 files changed, 27 insertions(+), 13 deletions(-)
setup.py | 41 ++++++++++++++++++++++++++-----------
2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/curated_tokenizers/_spp.pxd b/curated_tokenizers/_spp.pxd
index 29f5ec2..4630b45 100644
Expand All @@ -22,7 +22,7 @@ index 29f5ec2..4630b45 100644
uint32_t id() const
const string & piece() const
diff --git a/setup.py b/setup.py
index d0aa11b..0306041 100755
index d0aa11b..efa7ac1 100755
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,6 @@
Expand Down Expand Up @@ -53,11 +53,12 @@ index d0aa11b..0306041 100755
"-pthread",
"-DHAVE_PTHREAD=1",
],
@@ -139,21 +140,34 @@ def setup_package():
@@ -139,21 +140,37 @@ def setup_package():
if len(sys.argv) > 1 and sys.argv[1] == "clean":
return clean(root / "curated_tokenizers")

+ library_path = pathlib.Path(os.environ['PREFIX'])
+ extra_link_args = []
+ EXTENSION_LIBRARIES = ["sentencepiece"]
+ if "win32" in sys.platform:
+ absl_libs = (
Expand All @@ -67,6 +68,7 @@ index d0aa11b..0306041 100755
+ 'absl_flags_usage', 'absl_flags_usage_internal',
+ )
+ EXTENSION_LIBRARIES += ["libprotobuf"]
+ extra_link_args + ["/NODEFAULTLIB:libprotobuf.lib"]
+ library_path = library_path / "Library"
+ else:
+ absl_glob = pathlib.Path(os.environ['PREFIX']).glob('lib/libabsl_*.so')
Expand Down Expand Up @@ -94,6 +96,7 @@ index d0aa11b..0306041 100755
],
+ libraries=EXTENSION_LIBRARIES,
+ library_dirs=[str(library_path)],
+ extra_link_args=extra_link_args,
language="c++",
),
Extension(

0 comments on commit 5d63c4f

Please sign in to comment.