Solved compilation for macOS

cantoo-scribe · Jul 1, 2024 · a46d631 · a46d631
1 parent e6fbf91
commit a46d631
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 11 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,6 +2,38 @@ cmake_minimum_required(VERSION 3.10)
 
 project(MetalTranslate)
 
+include(ExternalProject)
+
+set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external)
+
+# Set ICU root directory (where Homebrew installed ICU)
+set(ICU_ROOT "/opt/homebrew/opt/icu4c")
+
+# Manually specify the include and library directories for ICU
+set(ICU_INCLUDE_DIR ${ICU_ROOT}/include)
+set(ICU_LIBRARY_DIR ${ICU_ROOT}/lib)
+
+# Find and link ICU
+find_package(ICU REQUIRED COMPONENTS uc i18n data)
+include_directories(${ICU_INCLUDE_DIR})
+
+ExternalProject_Add(CTranslate2
+    GIT_REPOSITORY https://github.com/OpenNMT/CTranslate2
+    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DWITH_MKL=OFF -DWITH_DNNL=ON -DOPENMP_RUNTIME=NONE
+)
+
+ExternalProject_Add(Tokenizer
+    GIT_REPOSITORY https://github.com/OpenNMT/Tokenizer
+    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION}
+    -DCMAKE_CXX_FLAGS="-Wno-enum-constexpr-conversion"
+    CXX_STANDARD 11
+)
+
+# -DCMAKE_CXX_FLAGS="-Wno-enum-constexpr-conversion"
+
+include_directories(${EXTERNAL_INSTALL_LOCATION}/include)
+link_directories(${EXTERNAL_INSTALL_LOCATION}/lib)
+
 add_library(metaltranslate SHARED
     src/MetalTranslate.cpp
     src/metal_api.cpp
@@ -22,10 +54,10 @@ target_include_directories(metaltranslate PRIVATE
 )
 
 # target_sources(metaltranslate PRIVATE src/MetalTranslate.cpp)
-add_subdirectory(third_party/CTranslate2)
+# add_subdirectory(third_party/CTranslate2)
 
 # target_link_libraries(metaltranslate ctranslate2)
-add_subdirectory(third_party/Tokenizer)
+# add_subdirectory(third_party/Tokenizer)
 
 # target_link_libraries(metaltranslate OpenNMTTokenizer)
 
@@ -34,4 +66,5 @@ target_link_libraries(metaltranslate PRIVATE ctranslate2 OpenNMTTokenizer)
 
 set_target_properties(metaltranslate PROPERTIES
     CXX_EXTENSIONS NO
+    CXX_STANDARD 17
 )
diff --git a/nodejs/index.js b/nodejs/index.js
@@ -8,7 +8,7 @@
 const { load, DataType, open, close, define } = require("ffi-rs");
 const path = require("path");
 
-const dynamicLib = "./libmetaltranslate.so";
+const dynamicLib = "./libmetaltranslate.dylib" //"./libmetaltranslate.so";
 // TODO: set dynamicLib according to OS
 
 open({
@@ -46,7 +46,7 @@ class Translator {
    */
   constructor(modelPath) {
     this.translator = metal.create_metal_translate([
-      modelPath + modelPath.endsWith("/") ? "" : "/",
+     modelPath,
     ]);
   }
 
@@ -73,8 +73,10 @@ class Translator {
 }
 
 // Example usage:
-/** 
-const modelPath = path.resolve("..", "models/translate-fairseq_m2m_100_418M/");
+
+const modelPath = path.resolve("..", "models/translate-fairseq_m2m_100_418M/", "model/");
+
+console.log('modelPath: ', modelPath)
 
 const testTranslator = new Translator(modelPath + "/");
 
@@ -85,6 +87,6 @@ const result = testTranslator.translate(textToTranslate, "en", "es");
 testTranslator.free();
 
 console.log("final result: ", result);
-*/
+
 
 exports.Translator = Translator;
diff --git a/src/MetalTranslate.cpp b/src/MetalTranslate.cpp
@@ -1,6 +1,6 @@
 #include "MetalTranslate.h"
 
-#include <ctranslate2/translator_pool.h>
+#include <ctranslate2/translator.h>
 #include <iostream>
 #include <onmt/Tokenizer.h>
 
@@ -17,6 +17,7 @@ std::string MetalTranslate::Translate(std::string source,
                                       std::string target_code) {
 
   // Tokenizer
+  std::cout << "model path " << this->_config.ModelPath << "\n"; 
   onmt::Tokenizer tokenizer(this->_config.ModelPath + "sentencepiece.model");
   std::vector<std::string> tokens;
   tokenizer.tokenize(source, tokens);
@@ -27,9 +28,10 @@ std::string MetalTranslate::Translate(std::string source,
   // CTranslate2
   const size_t num_translators = 1;
   const size_t num_threads_per_translator = 0; // Unused with DNNL
-  ctranslate2::TranslatorPool translator(
-      num_translators, num_threads_per_translator,
-      this->_config.ModelPath + "model", ctranslate2::Device::CPU);
+  // ctranslate2::Translator translator(
+  //     num_translators, num_threads_per_translator,
+  //     this->_config.ModelPath + "model", ctranslate2::Device::CPU);
+  ctranslate2::Translator translator(this->_config.ModelPath, ctranslate2::Device::CPU);
 
   const std::vector<std::vector<std::string>> batch = {tokens};
   const std::vector<std::vector<std::string>> target_prefix = {