Add basic iOS phi-3 sample (#433)

* initial ios phi-3 app

* update to remove tests etc.

* update brief read me

* minor update

* update

* address pr comments

* minor update

* minor updates

* remove the unnecessary delay step

* Update mobile/examples/phi-3/ios/LocalLLM/LocalLLM/README.md

Co-authored-by: Scott McKay <[email protected]>

* address pr comments

* pr comments

* minor update

* pr comments

* update

* Update mobile/examples/phi-3/ios/LocalLLM/LocalLLM/README.md

Co-authored-by: Scott McKay <[email protected]>

---------

Co-authored-by: rachguo <[email protected]>
Co-authored-by: Scott McKay <[email protected]>

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,13 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "platform" : "ios",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/ContentView.swift

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import SwiftUI
+
+struct ContentView: View {
+    @ObservedObject var tokenUpdater = SharedTokenUpdater.shared
+
+    var body: some View {
+        VStack {
+            ScrollView {
+                VStack(alignment: .leading) {
+                    ForEach(tokenUpdater.decodedTokens, id: \.self) { token in
+                        Text(token)
+                           .padding(.horizontal, 5)
+                    }
+                }
+                .padding()
+            }
+            Button("Generate Tokens") {
+                DispatchQueue.global(qos: .background).async {
+                    // TODO: add user prompt question UI
+                    GenAIGenerator.generate("Who is the current US president?");
+                }
+            }
+        }
+    }
+}
+
+struct ContentView_Previews: PreviewProvider {
+    static var previews: some View {
+        ContentView()
+    }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/GenAIGenerator.h

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#ifndef GenAIGenerator_h
+#define GenAIGenerator_h
+
+#import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface GenAIGenerator : NSObject
+
++ (void)generate:(NSString *)input_user_question;
+
+@end
+
+NS_ASSUME_NONNULL_END
+
+#endif /* GenAIGenerator_h */

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/GenAIGenerator.mm

@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#import "GenAIGenerator.h"
+#include "LocalLLM-Swift.h"
+#include "ort_genai.h"
+#include "ort_genai_c.h"
+
+
+@implementation GenAIGenerator
+
++ (void)generate:(nonnull NSString*)input_user_question {
+  NSString* llmPath = [[NSBundle mainBundle] resourcePath];
+  const char* modelPath = llmPath.cString;
+
+  auto model = OgaModel::Create(modelPath);
+  auto tokenizer = OgaTokenizer::Create(*model);
+
+  NSString* promptString = [NSString stringWithFormat:@"<|user|>\n%@<|end|>\n<|assistant|>", input_user_question];
+  const char* prompt = [promptString UTF8String];
+
+  auto sequences = OgaSequences::Create();
+  tokenizer->Encode(prompt, *sequences);
+
+  auto params = OgaGeneratorParams::Create(*model);
+  params->SetSearchOption("max_length", 200);
+  params->SetInputSequences(*sequences);
+
+  // Streaming Output to generate token by token
+  auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer);
+
+  auto generator = OgaGenerator::Create(*model, *params);
+
+  while (!generator->IsDone()) {
+    generator->ComputeLogits();
+    generator->GenerateNextToken();
+
+    const int32_t* seq = generator->GetSequenceData(0);
+    size_t seq_len = generator->GetSequenceCount(0);
+    const char* decode_tokens = tokenizer_stream->Decode(seq[seq_len - 1]);
+
+    NSLog(@"Decoded tokens: %s", decode_tokens);
+
+    // Add decoded token to SharedTokenUpdater
+    NSString* decodedTokenString = [NSString stringWithUTF8String:decode_tokens];
+    [SharedTokenUpdater.shared addDecodedToken:decodedTokenString];
+  }
+}
+@end

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/LocalLLM-Bridging-Header.h

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#ifndef LocalLLM_Bridging_Header_h
+#define LocalLLM_Bridging_Header_h
+
+#import "GenAIGenerator.h"
+
+#endif /* LocalLLM_Bridging_Header_h */

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/LocalLLMApp.swift

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+
+import SwiftUI
+
+@main
+struct LocalLLMApp: App {
+    var body: some Scene {
+        WindowGroup {
+            ContentView()
+        }
+    }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Preview Content/Preview Assets.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/README.md

@@ -0,0 +1,109 @@
+# **Local LLM sample application running Phi3-mini on iOS**
+
+## **Steps**
+
+### General prerequisites
+
+See the general prerequisites [here](../../../../../README.md#General-Prerequisites).
+
+For this application, the following prerequisites are preferred:
+
+1. macOS 14+
+
+2. Xcode 15+ (latest Xcode version perferred.)
+
+3. iOS SDK 16.x + (iPhone 14 or iPhone 15 powered by a A16 or A17 preferred)
+
+**Note**: 
+  The current Xcode project contains a built .dylib for ORT and ORT GenAI. The following steps `A, B, C` under `step 1.` for building from source for the libraries are optional.
+  However if you want to build from source to include the latest updates, please use the `step 1.` as a reference.
+
+### 1. Steps to build from source for ONNX Runtime and Generative AI libraries [Optional]
+
+#### **A. Preparation**
+
+   - Install Python 3.10+
+
+   - Install flatbuffers
+     ```
+       pip3 install flatbuffers
+     ```
+
+   - Install [CMake](https://cmake.org/download/)
+
+#### **B. Compiling ONNX Runtime for iOS**
+
+```bash
+
+git clone https://github.com/microsoft/onnxruntime.git
+
+cd onnxruntime
+
+./build.sh --build_shared_lib --skip_tests --parallel --build_dir ./build_ios --ios --apple_sysroot iphoneos --osx_arch arm64 --apple_deploy_target 16.6 --cmake_generator Xcode --config Release
+
+```
+
+***Notice***
+
+  1. Before compiling, you must ensure that Xcode is configured correctly and set it on the terminal
+
+```bash
+
+sudo xcode-select -switch /Applications/Xcode.app/Contents/Developer 
+
+```
+
+  2. ONNX Runtime needs to be compiled based on different platforms. For iOS, you can compile for arm64 or x86_64 based on needs. If you are running an iOS simulator on an Intel mac, compile for x86_64. Use arm64 for an ARM based mac to run the simulator, and to run on an iPhone.
+
+  3. It is recommended to directly use the latest iOS SDK for compilation. Of course, you can also lower the version to be compatible with past SDKs.
+
+#### **C. Compiling Generative AI with ONNX Runtime for iOS**
+
+```bash
+
+git clone https://github.com/microsoft/onnxruntime-genai
+
+cd onnxruntime-genai
+
+python3 build.py --parallel --build_dir ./build_iphoneos --ios --ios_sysroot iphoneos --ios_arch arm64 --ios_deployment_target 16.6 --cmake_generator Xcode
+
+```
+
+#### **D. Copy over latest header files and required .dylibs built from source**
+
+If you build from source and get the latest .dylibs for ORT and ORT GenAI, please copy the .dylibs over to `mobile\examples\phi-3\ios\LocalLLM\LocalLLM\lib` and copy the latest header files over to `mobile\examples\phi-3\ios\LocalLLM\LocalLLM\header` 
+
+The build output path for libonnxruntime.dylib is `<ORT_PROJECT_ROOT>/build/intermediates/<platform>_<arch>/<build_config>/<build_config-platform>/libonnxruntime.dylib` 
+The build output path for libonnxruntime-genai.dylib is `<ORT_GENAI_PROJECT_ROOT>/build/<build_config-platform>/libonnxruntime-genai.dylib`. 
+
+For example: 
+- `onnxruntime/build/intermediates/iphoneos_arm64/Release/Release-iphoneos/libonnxruntime.1.19.0.dylib`
+- `onnxruntime-genai/build/Release/Release-iphoneos/libonnxruntime-genai.dylib`.
+
+Note that you will need to build and copy the correct dylib for the target architecture you wish to run the app on.
+e.g. 
+if you want to run on the iOS simulator on an Intel mac, you must build both onnxruntime and onnxruntime-genai for x86_64 and copy the dylibs to the app's `lib` directory.
+if you want to run on an iPhone, you must build both onnxruntime and onnxruntime-genai for arm64 and copy the dylibs to the app's `lib` directory.
+
+The header files to copy are:
+`<ORT_MAIN_SOURCE_REPO>/onnxruntime/core/session/onnxruntime_c_api.h`,
+`<ORT_GENAI_MAIN_SOURCE_REPO>/src/ort_genai.h`,
+`<ORT_GENAI_MAIN_SOURCE_REPO>/src/ort_genai_c.h`.
+
+### 2. Create/Open the iOS application in Xcode
+
+The app uses Objective-C/C++ since using Generative AI with ONNX Runtime C++ API, Objective-C has better compatiblility.
+
+### 3. Copy the ONNX quantized INT4 model to the App application project
+
+Download from hf repo: <https://huggingface.co/microsoft/Phi-3-mini-128k-instruct-onnx/tree/main/cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4>
+
+After downloading completes, you need to copy files over to the `Resources` directory in the `Destination` column of `Target-LocalLLM`->`Build Phases`-> `New Copy File Phases` -> `Copy Files`.
+
+Upon app launching, Xcode will automatically copy and install the model files from Resources folder and directly download to the iOS device.
+
+### 4. Run the app and checkout the streaming output token results
+
+**Note**: The current app only sets up with a simple initial prompt question, you can adjust/try your own or refine the UI based on requirements.
+
+***Notice:*** The current Xcode project runs on iOS 16.6, feel free to adjust latest iOS/build for lates iOS versions accordingly.

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/SharedTokenUpdater.swift

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import Combine
+import Foundation
+
+@objc class SharedTokenUpdater: NSObject, ObservableObject {
+    @Published var decodedTokens: [String] = []
+
+    @objc static let shared = SharedTokenUpdater()
+
+    @objc func addDecodedToken(_ token: String) {
+        DispatchQueue.main.async {
+            self.decodedTokens.append(token)
+        }
+    }
+}

mobile/examples/phi-3/ios/LocalLLM/LocalLLM/header/README.md

@@ -0,0 +1,2 @@
+**Note**:
+This folder contains the latest C++ headers that's required for the project. Copied over from the ORT and ORT GenAI repo matches the latest build.