From 10a4360b2fa0f79f63def3f2fc945221a5bb8645 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 18 Jul 2025 07:42:51 -0400 Subject: [PATCH] handle sentencepiece Signed-off-by: Vladimir Mandic --- installer.py | 23 +++++++++++++++-------- launch.py | 1 + 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/installer.py b/installer.py index fec4367ec..bf684f527 100644 --- a/installer.py +++ b/installer.py @@ -1226,6 +1226,21 @@ def install_optional(): ts('optional', t_start) +def install_sentencepipece(): + if installed('sentencepiece', quiet=True): + pass + elif int(sys.version_info.minor) >= 13: + backup_cmake_policy = os.environ.get('CMAKE_POLICY_VERSION_MINIMUM', None) + backup_cxxflags = os.environ.get('CXXFLAGS', None) + os.environ.setdefault('CMAKE_POLICY_VERSION_MINIMUM', '3.5') + os.environ.setdefault('CXXFLAGS', '-include cstdint') + install('git+https://github.com/google/sentencepiece#subdirectory=python', 'sentencepiece') + os.environ.setdefault('CMAKE_POLICY_VERSION_MINIMUM', backup_cmake_policy) + os.environ.setdefault('CXXFLAGS', backup_cxxflags) + else: + install('sentencepiece', 'sentencepiece') + + def install_requirements(): t_start = time.time() if args.profile: @@ -1243,14 +1258,6 @@ def install_requirements(): if args.optional: quick_allowed = False install_optional() - # gcc 15 patch for sentencepiece - backup_cmake_policy = os.environ.get('CMAKE_POLICY_VERSION_MINIMUM', None) - backup_cxxflags = os.environ.get('CXXFLAGS', None) - os.environ.setdefault('CMAKE_POLICY_VERSION_MINIMUM', '3.5') - os.environ.setdefault('CXXFLAGS', '-include cstdint') - install('git+https://github.com/google/sentencepiece#subdirectory=python', 'sentencepiece') - os.environ.setdefault('CMAKE_POLICY_VERSION_MINIMUM', backup_cmake_policy) - os.environ.setdefault('CXXFLAGS', backup_cxxflags) installed('torch', reload=True) # reload packages cache log.info('Install: verifying requirements') with open('requirements.txt', 'r', encoding='utf8') as f: diff --git a/launch.py b/launch.py index 198eb03e0..bb28a82fb 100755 --- a/launch.py +++ b/launch.py @@ -265,6 +265,7 @@ def main(): installer.check_onnx() installer.check_transformers() installer.check_diffusers() + installer.install_sentencepipece() installer.check_modified_files() if args.test: installer.log.info('Startup: test mode')