May Binary Update

.github/workflows/compile.yml

@@ -28,13 +28,25 @@ jobs:
         include:
           - build: 'noavx'
             defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF'
+            os: ubuntu-24.04
+            arch: x64
           - build: 'avx2'
             defines: ''
+            os: ubuntu-24.04
+            arch: x64
           - build: 'avx'
             defines: '-DGGML_AVX2=OFF'
+            os: ubuntu-24.04
+            arch: x64
           - build: 'avx512'
             defines: '-DGGML_AVX512=ON'
-    runs-on: ubuntu-24.04
+            os: ubuntu-24.04
+            arch: x64
+          - build: 'aarch64'
+            defines: '-DGGML_NATIVE=OFF -DGGML_CPU_AARCH64=ON -DGGML_CPU_ARM_ARCH=armv8-a'
+            os: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -52,28 +64,28 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           path: ./build/bin/libllama.so
-          name: llama-bin-linux-${{ matrix.build }}-x64.so
+          name: llama-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
           if-no-files-found: error
       - uses: actions/upload-artifact@v4
         with:
           path: ./build/bin/libggml.so
-          name: ggml-bin-linux-${{ matrix.build }}-x64.so
+          name: ggml-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
           if-no-files-found: error
       - uses: actions/upload-artifact@v4
         with:
           path: ./build/bin/libggml-base.so
-          name: ggml-base-bin-linux-${{ matrix.build }}-x64.so
+          name: ggml-base-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
           if-no-files-found: error
       - uses: actions/upload-artifact@v4
         with:
           path: ./build/bin/libggml-cpu.so
-          name: ggml-cpu-bin-linux-${{ matrix.build }}-x64.so
+          name: ggml-cpu-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
           if-no-files-found: error
       - name: Upload Llava
         uses: actions/upload-artifact@v4
         with:
           path: ./build/bin/libllava_shared.so
-          name: llava-bin-linux-${{ matrix.build }}-x64.so
+          name: llava-bin-linux-${{ matrix.build }}-${{ matrix.arch }}.so
           if-no-files-found: error
 
   compile-musl:
@@ -527,19 +539,15 @@ jobs:
           if-no-files-found: error
 
   compile-android:
-    # Disable android build
-    if: false
-
+    name: Compile (Android)
     strategy:
       fail-fast: true
       matrix:
         include:
-          - build: 'x86'
-            defines: '-DANDROID_ABI=x86'
           - build: 'x86_64'
-            defines: '-DANDROID_ABI=x86_64'
+            defines: '-DANDROID_ABI=x86_64 -DCMAKE_C_FLAGS=-march=x86-64 -DCMAKE_CXX_FLAGS=-march=x86-64'
           - build: 'arm64-v8a'
-            defines: '-DANDROID_ABI=arm64-v8a'
+            defines: '-DANDROID_ABI=arm64-v8a -DCMAKE_C_FLAGS=-march=armv8.7a -DCMAKE_C_FLAGS=-march=armv8.7a'
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
@@ -555,28 +563,39 @@ jobs:
       - name: Build
         id: cmake_build
         env:
-          CMAKE_FLAGS: '-DCMAKE_TOOLCHAIN_FILE=${{ steps.setup-ndk.outputs.ndk-path }}/build/cmake/android.toolchain.cmake -DANDROID_PLATFORM=android-23'
+          CMAKE_FLAGS: '-DCMAKE_TOOLCHAIN_FILE=${{ steps.setup-ndk.outputs.ndk-path }}/build/cmake/android.toolchain.cmake -DANDROID_PLATFORM=android-23 -DGGML_OPENMP=OFF -DGGML_LLAMAFILE=OFF'
         run: |
-          mkdir build
-          cd build
-          cmake .. ${{ env.COMMON_DEFINE }} ${{ env.CMAKE_FLAGS }} ${{ matrix.defines }}
-          cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS}
-          cd ..
-          ls -R
+          # export-lora not supported on 32 bit machines hence breaks x86 build
+          sed -i '/add_subdirectory(export-lora)/d' examples/CMakeLists.txt # remove export-lora from examples
+          cmake ${{ env.COMMON_DEFINE }} ${{ env.CMAKE_FLAGS }} ${{ matrix.defines }} -B build
+          cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
       - name: Upload Llama
         uses: actions/upload-artifact@v4
         with:
-          path: ./build/src/libllama.so
+          path: ./build/bin/libllama.so
           name: llama-bin-android-${{ matrix.build }}.so
-      - uses: actions/upload-artifact@v4
+      - name: Upload GGML
+        uses: actions/upload-artifact@v4
         with:
-          path: ./build/ggml/src/libggml.so
+          path: ./build/bin/libggml.so
           name: ggml-bin-android-${{ matrix.build }}.so
           if-no-files-found: error
+      - name: Upload GGML Base
+        uses: actions/upload-artifact@v4
+        with:
+          path: ./build/bin/libggml-base.so
+          name: ggml-base-bin-android-${{ matrix.build }}.so
+          if-no-files-found: error
+      - name: Upload GGML CPU
+        uses: actions/upload-artifact@v4
+        with:
+          path: ./build/bin/libggml-cpu.so
+          name: ggml-cpu-bin-android-${{ matrix.build }}.so
+          if-no-files-found: error
       - name: Upload Llava
         uses: actions/upload-artifact@v4
         with:
-          path: ./build/examples/llava/libllava_shared.so
+          path: ./build/bin/libllava_shared.so
           name: llava-bin-android-${{ matrix.build }}.so
 
   build-deps:
@@ -601,7 +620,7 @@ jobs:
       - name: Rearrange Files
         run: |
           # Make all directories at once
-          mkdir --parents deps/{noavx,avx,avx2,avx512,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
+          mkdir --parents deps/{noavx,avx,avx2,avx512,linux-arm64,musl-noavx,musl-avx,musl-avx2,musl-avx512,osx-arm64,osx-x64,osx-x64-rosetta2,cu11.7.1,cu12.2.0,vulkan,android-arm64-v8a,android-x86,android-x86_64}
 
           # Linux
           cp artifacts/ggml-bin-linux-noavx-x64.so/libggml.so           deps/noavx/libggml.so
@@ -628,6 +647,13 @@ jobs:
           cp artifacts/llama-bin-linux-avx512-x64.so/libllama.so         deps/avx512/libllama.so
           cp artifacts/llava-bin-linux-avx512-x64.so/libllava_shared.so  deps/avx512/libllava_shared.so
 
+          # Arm64
+          cp artifacts/ggml-bin-linux-aarch64-arm64.so/libggml.so           deps/linux-arm64/libggml.so
+          cp artifacts/ggml-base-bin-linux-aarch64-arm64.so/libggml-base.so deps/linux-arm64/libggml-base.so
+          cp artifacts/ggml-cpu-bin-linux-aarch64-arm64.so/libggml-cpu.so   deps/linux-arm64/libggml-cpu.so
+          cp artifacts/llama-bin-linux-aarch64-arm64.so/libllama.so         deps/linux-arm64/libllama.so
+          cp artifacts/llava-bin-linux-aarch64-arm64.so/libllava_shared.so  deps/linux-arm64/libllava_shared.so
+
           # Musl
           cp artifacts/ggml-bin-musl-noavx-x64.so/libggml.so           deps/musl-noavx/libggml.so
           cp artifacts/ggml-base-bin-musl-noavx-x64.so/libggml-base.so deps/musl-noavx/libggml-base.so
@@ -703,17 +729,17 @@ jobs:
           cp artifacts/llava-bin-osx-x64-rosetta2.dylib/libllava_shared.dylib  deps/osx-x64-rosetta2/libllava_shared.dylib
 
           # Android
-          #cp artifacts/ggml-bin-android-arm64-v8a.so/libggml.so           deps/android-arm64-v8a/libggml.so
-          #cp artifacts/llama-bin-android-arm64-v8a.so/libllama.so         deps/android-arm64-v8a/libllama.so
-          #cp artifacts/llava-bin-android-arm64-v8a.so/libllava_shared.so  deps/android-arm64-v8a/libllava_shared.so
-
-          #cp artifacts/ggml-bin-android-x86.so/libggml.so                 deps/android-x86/libggml.so
-          #cp artifacts/llama-bin-android-x86.so/libllama.so               deps/android-x86/libllama.so
-          #cp artifacts/llava-bin-android-x86.so/libllava_shared.so        deps/android-x86/libllava_shared.so
-
-          #cp artifacts/ggml-bin-android-x86_64.so/libggml.so              deps/android-x86_64/libggml.so
-          #cp artifacts/llama-bin-android-x86_64.so/libllama.so            deps/android-x86_64/libllama.so
-          #cp artifacts/llava-bin-android-x86_64.so/libllava_shared.so     deps/android-x86_64/libllava_shared.so
+          cp artifacts/ggml-bin-android-arm64-v8a.so/libggml.so               deps/android-arm64-v8a/libggml.so
+          cp artifacts/ggml-base-bin-android-arm64-v8a.so/libggml-base.so     deps/android-arm64-v8a/libggml-base.so
+          cp artifacts/ggml-cpu-bin-android-arm64-v8a.so/libggml-cpu.so       deps/android-arm64-v8a/libggml-cpu.so
+          cp artifacts/llama-bin-android-arm64-v8a.so/libllama.so             deps/android-arm64-v8a/libllama.so
+          cp artifacts/llava-bin-android-arm64-v8a.so/libllava_shared.so      deps/android-arm64-v8a/libllava_shared.so
+                                                                              
+          cp artifacts/ggml-bin-android-x86_64.so/libggml.so                  deps/android-x86_64/libggml.so
+          cp artifacts/ggml-base-bin-android-x86_64.so/libggml-base.so        deps/android-x86_64/libggml-base.so
+          cp artifacts/ggml-cpu-bin-android-x86_64.so/libggml-cpu.so          deps/android-x86_64/libggml-cpu.so
+          cp artifacts/llama-bin-android-x86_64.so/libllama.so                deps/android-x86_64/libllama.so
+          cp artifacts/llava-bin-android-x86_64.so/libllava_shared.so         deps/android-x86_64/libllava_shared.so
 
           # Windows CUDA
           cp artifacts/ggml-bin-win-cublas-cu11.7.1-x64.dll/ggml.dll            deps/cu11.7.1/ggml.dll

.github/workflows/main.yml

@@ -38,6 +38,15 @@ jobs:
       with:
         dotnet-version: |
           8.0.x
+    - name: Install Mobile Workloads
+      if: ${{ contains(runner.os, 'windows') }}
+      run: |
+        dotnet workload install android --ignore-failed-sources
+        dotnet workload install maui --ignore-failed-sources
+    - name: Remove Mobile Project
+      if: ${{ !contains(runner.os, 'windows') }}
+      run: |
+        dotnet sln LLamaSharp.sln remove Llama.Mobile
     - name: Cache Packages
       uses: actions/cache@v4
       with:

.gitignore

@@ -337,7 +337,6 @@ test/TensorFlowNET.Examples/mnist
 # training model resources
 .resources
 /redist
-*.xml
 *.xsd
 
 # docs

LLama/LLamaSharp.Runtime.targets

@@ -202,6 +202,28 @@
       </None>
 
 
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libllama.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-arm64/native/libllama.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-arm64/native/libggml.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml-base.so">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+          <Link>runtimes/linux-arm64/native/libggml-base.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libggml-cpu.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-arm64/native/libggml-cpu.so</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/linux-arm64/libllava_shared.so">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        <Link>runtimes/linux-arm64/native/libllava_shared.so</Link>
+      </None>
+
+
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/cu11.7.1/libllama.so">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>
@@ -466,4 +488,94 @@
         <Link>runtimes/linux-x64/native/vulkan/libllava_shared.so</Link>
       </None>
     </ItemGroup>
+
+    <!-- Android Native Libs (Start) -->
+    <ItemGroup
+        Condition="$(AndroidSupportedAbis.Contains('x86')) or $(RuntimeIdentifiers.Contains('android-x86'))">
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libllama.so">
+            <Link>runtimes/android-x86/native/libllama.so</Link>
+            <Abi>x86</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libggml.so">
+            <Link>runtimes/android-x86/native/libggml.so</Link>
+            <Abi>x86</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libggml-base.so">
+            <Link>runtimes/android-x86/native/libggml-base.so</Link>
+            <Abi>x86</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libggml-cpu.so">
+            <Link>runtimes/android-x86/native/libggml-cpu.so</Link>
+            <Abi>x86</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86/libllava_shared.so">
+            <Link>runtimes/android-x86/native/libllava_shared.so</Link>
+            <Abi>x86</Abi>
+        </AndroidNativeLibrary>
+    </ItemGroup>
+
+    <ItemGroup
+        Condition="$(AndroidSupportedAbis.Contains('x86_64')) or $(RuntimeIdentifiers.Contains('android-x64'))">
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libllama.so">
+            <Link>lib/x86_64/libllama.so</Link>
+            <Abi>x86_64</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libggml.so">
+            <Link>lib/x86_64/libggml.so</Link>
+            <Abi>x86_64</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libggml-base.so">
+            <Link>lib/x86_64/libggml-base.so</Link>
+            <Abi>x86_64</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libggml-cpu.so">
+            <Link>lib/x86_64/libggml-cpu.so</Link>
+            <Abi>x86_64</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-x86_64/libllava_shared.so">
+            <Link>lib/x86_64/libllava_shared.so</Link>
+            <Abi>x86_64</Abi>
+        </AndroidNativeLibrary>
+    </ItemGroup>
+
+    <ItemGroup
+        Condition="$(AndroidSupportedAbis.Contains('arm64-v8a')) or $(RuntimeIdentifiers.Contains('android-arm64'))">
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libllama.so">
+            <Link>lib/arm64-v8a/libllama.so</Link>
+            <Abi>arm64-v8a</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libggml.so">
+            <Link>lib/arm64-v8a/libggml.so</Link>
+            <Abi>arm64-v8a</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libggml-base.so">
+            <Link>lib/arm64-v8a/libggml-base.so</Link>
+            <Abi>arm64-v8a</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libggml-cpu.so">
+            <Link>lib/arm64-v8a/libggml-cpu.so</Link>
+            <Abi>arm64-v8a</Abi>
+        </AndroidNativeLibrary>
+        <AndroidNativeLibrary Visible="false"
+            Include="$(MSBuildThisFileDirectory)runtimes/deps/android-arm64-v8a/libllava_shared.so">
+            <Link>lib/arm64-v8a/libllava_shared.so</Link>
+            <Abi>arm64-v8a</Abi>
+        </AndroidNativeLibrary>
+    </ItemGroup>
+    <!-- Android Native Libs (End) -->
+
 </Project>

LLama/LLamaSharp.csproj

@@ -57,7 +57,7 @@
   </ItemGroup>
 
   <PropertyGroup>
-    <BinaryReleaseId>be7c3034108473be</BinaryReleaseId>
+    <BinaryReleaseId>ceda28ef8e310a8de</BinaryReleaseId>
   </PropertyGroup>
 
   <PropertyGroup>

LLama/Native/LLamaModelParams.cs

@@ -13,6 +13,11 @@ public unsafe struct LLamaModelParams
         /// todo: add support for llama_model_params.devices
         /// </summary>
         private IntPtr devices;
+
+        // NULL-terminated list of buffer types to use for tensors that match a pattern
+        // actual type: llama_model_tensor_buft_override* 
+        // todo: add support for tensor_buft_overrides
+        private IntPtr tensor_buft_overrides;
 
         /// <summary>
         /// // number of layers to store in VRAM

LLama/Native/LLamaModelQuantizeParams.cs

@@ -89,6 +89,11 @@ public bool keep_split
         /// </summary>
         public IntPtr kv_overrides;
 
+        /// <summary>
+        /// pointer to vector containing tensor types
+        /// </summary>
+        public IntPtr tensor_types;
+
         /// <summary>
         /// Create a LLamaModelQuantizeParams with default values
         /// </summary>

LLama/Native/LLamaVocabPreType.cs

@@ -38,5 +38,10 @@ internal enum LLamaVocabPreType
     MINERVA = 27,
     DEEPSEEK3_LLM = 28,
     GPT4O = 29,
+    SUPERBPE = 30,
+    TRILLION = 31,
+    BAILINGMOE = 32,
+    LLAMA4 = 33,
+    PIXTRAL = 34,
 }
 // ReSharper restore InconsistentNaming