Initial port of april-asr to use ggml

abb128 · Jan 27, 2025 · cd6b9e0 · cd6b9e0
1 parent 014be57
commit cd6b9e0
Show file tree

Hide file tree

Showing 27 changed files with 1,758 additions and 1,550 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/ggml"]
+	path = src/ggml
+	url = https://github.com/ggerganov/ggml.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -11,89 +11,9 @@ project(april-asr
 
 string(REPLACE "-" "" PROJECT_NAME_SO ${PROJECT_NAME})
 
-if(WIN32)
-    set(CMAKE_SHARED_LIBRARY_PREFIX "lib")
-    set(CMAKE_STATIC_LIBRARY_PREFIX "lib")
-endif()
+add_subdirectory(src/ggml)
 
 cmake_policy(SET CMP0074 NEW)
-if (DEFINED ENV{ONNX_ROOT})
-  set(ONNXRuntime_ROOT_DIR $ENV{ONNX_ROOT})
-else()
-  set(ONNXRuntime_ROOT_DIR ${PROJECT_SOURCE_DIR}/lib)
-endif()
-
-# To statically link, set environment variable like so (example):
-#  ONNX_STATIC_ROOT=$HOME/Documents/onnxruntime/build/Linux/RelWithDebInfo
-if (NOT DEFINED ENV{ONNX_STATIC_ROOT})
-  find_package(ONNXRuntime REQUIRED)
-  if(NOT ONNXRuntime_FOUND)
-    message(FATAL_ERROR "Could not find ONNXRuntime")
-  endif()
-
-  set(april_link_libraries ONNXRuntime::ONNXRuntime)
-else()
-  set(april_link_libraries
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_session.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_optimizer.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_providers.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_framework.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_graph.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_util.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_mlas.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_common.a
-    $ENV{ONNX_STATIC_ROOT}/libonnxruntime_flatbuffers.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/onnx-build/libonnx.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/onnx-build/libonnx_proto.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/protobuf-build/libprotobuf-lite.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/re2-build/libre2.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/flatbuffers-build/libflatbuffers.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_base.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_throw_delegate.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/container/libabsl_raw_hash_set.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/hash/libabsl_hash.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/hash/libabsl_city.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/hash/libabsl_low_level_hash.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_raw_logging_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/google_nsync-build/libnsync_cpp.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/pytorch_cpuinfo-build/libcpuinfo.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/pytorch_cpuinfo-build/deps/clog/libclog.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_cord.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_cordz_info.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_cord_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_cordz_functions.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_cordz_handle.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/container/libabsl_raw_hash_set.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/container/libabsl_hashtablez_sampler.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/profiling/libabsl_exponential_biased.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/synchronization/libabsl_synchronization.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/synchronization/libabsl_graphcycles_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/debugging/libabsl_stacktrace.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/debugging/libabsl_symbolize.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_malloc_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/debugging/libabsl_debugging_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/debugging/libabsl_demangle_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/time/libabsl_time.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/time/libabsl_civil_time.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/time/libabsl_time_zone.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/hash/libabsl_hash.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/types/libabsl_bad_optional_access.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_strings.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_throw_delegate.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/strings/libabsl_strings_internal.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/types/libabsl_bad_variant_access.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/hash/libabsl_city.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/hash/libabsl_low_level_hash.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_base.a
-    $ENV{ONNX_STATIC_ROOT}/_deps/abseil_cpp-build/absl/base/libabsl_spinlock_wait.a
-    dl
-    rt
-  )
-
-  set(ORT_INCLUDE_DIR
-    $ENV{ONNX_STATIC_ROOT}/../../../include/onnxruntime/core/session/
-  )
-endif()
 
 if (DEFINED ENV{APRIL_DEBUG_SAVE_AUDIO})
     add_compile_definitions(APRIL_DEBUG_SAVE_AUDIO)
@@ -107,10 +27,10 @@ set(april_sources
   src/proc_thread.c
   src/params.c
   src/fbank.c
-  src/ort_util.c
   src/file/model_file.c
   src/fft/pocketfft.c
   src/sonic/sonic.c
+  src/model_impl.cpp
 )
 
 file(GLOB_RECURSE april_headers "*.h")
@@ -133,13 +53,13 @@ if(NOT WIN32)
   list(APPEND april_link_libraries "m")
 endif()
 
-include_directories(${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/src ${ORT_INCLUDE_DIR})
+include_directories(${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/ggml/include)
 
 add_library(aprilasr_static STATIC ${april_sources})
-target_link_libraries(aprilasr_static ${april_link_libraries})
+target_link_libraries(aprilasr_static ggml ${april_link_libraries})
 
 add_library(aprilasr SHARED ${april_sources})
-target_link_libraries(aprilasr ${april_link_libraries})
+target_link_libraries(aprilasr ggml ${april_link_libraries})
 
 set_target_properties(aprilasr PROPERTIES VERSION ${CMAKE_PROJECT_VERSION}
 SOVERSION ${PROJECT_VERSION_MAJOR} )
@@ -151,6 +71,12 @@ target_link_libraries(main PRIVATE aprilasr_static ${april_link_libraries})
 add_executable(srt example_srt.cpp)
 target_link_libraries(srt PRIVATE aprilasr_static ${april_link_libraries})
 
+add_executable(stress rt_stress.cpp)
+target_link_libraries(stress PRIVATE aprilasr_static ${april_link_libraries})
+
+add_executable(quantize src/tools/quantize.cpp)
+target_link_libraries(quantize PRIVATE aprilasr_static ${april_link_libraries})
+
 install(TARGETS aprilasr
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
   PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})

diff --git a/README.md b/README.md
@@ -31,27 +31,6 @@ Currently only one model is available, the [English model](https://april.sapples
 To make your own models, check out `extra/exporting-howto.md`
 
 ## Building on Linux
-Building requires ONNXRuntime v1.13.1. You can either try to build it from source or just download the release binaries.
-
-### Downloading ONNXRuntime
-Run `./download_onnx_linux_x64.sh` for linux-x64.
-
-For other platforms the script should be very similar, or visit https://github.com/microsoft/onnxruntime/releases/tag/v1.13.1 and download the right zip/tgz file for your platform and extract the contents to a directory named `lib`.
-
-You may also define the env variable `ONNX_ROOT` containing a path to where you extracted the archive, if placing it in `lib` isn't a choice.
-
-### Building ONNXRuntime from source (untested)
-You don't need to do this if you've downloaded ONNXRuntime.
-
-Follow the instructions here: https://onnxruntime.ai/docs/how-to/build/inferencing.html#linux
-
-then run
-```
-cd build/Linux/RelWithDebInfo/
-sudo make install
-```
-
-### Building aprilasr
 Run:
 ```
 $ mkdir build
@@ -60,33 +39,21 @@ $ cmake -DCMAKE_BUILD_TYPE=Release ..
 $ make -j4
 ```
 
-You should now have `main`, `libaprilasr.so` and `libaprilasr_static.so`.
-
-If running `main` fails because it can't find `libonnxruntime.so.1.13.1`, you may need to make `libonnxruntime.so.1.13.1` accessible like so:
-```
-$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/../lib/lib/
-```
+You should now have `main` and `libaprilasr.so`
 
 ## Building on Windows (msvc)
-Create a folder called `lib` in the april-asr folder.
-
-Download [onnxruntime-win-x64-1.13.1.zip](https://github.com/microsoft/onnxruntime/releases/download/v1.13.1/onnxruntime-win-x64-1.13.1.zip) and extract the insides of the onnxruntime-win-x64-1.13.1 folder to the `lib` folder
-
-Run cmake to configure and generate Visual Studio project files. Make sure you select x64 as the target if you have downloaded the x64 version of ONNXRuntime.
+Run cmake to configure and generate Visual Studio project files.
 
 Open the `ALL_BUILD.vcxproj` and everything should build. The output will be in the Release or Debug folders.
 
-When running main.exe you may receive an error message like this:
-> The application was unable to start correctly (0xc000007b)
-
-To fix this, you need to make onnxruntime.dll available. One way to do this is to copy onnxruntime.dll from lib/lib/onnxruntime.dll to build/Debug and build/Release. You may need to distribute the dll together with your application.
-
 ## Applications
 Currently I'm developing [Live Captions](https://github.com/abb128/LiveCaptions), a Linux desktop app that provides live captioning.
 
 ## Acknowledgements
 Thanks to the [k2-fsa/icefall](https://github.com/k2-fsa/icefall) contributors for creating the speech recognition recipes and models.
 
+Thanks to the developers of [ggml](https://github.com/ggerganov/ggml) and [llama.cpp](https://github.com/ggerganov/llama.cpp), who have created a performant, lightweight and flexible C tensor library.
+
 This project makes use of a few libraries:
 * pocketfft, authored by Martin Reinecke, Copyright (C) 2008-2018 Max-Planck-Society, licensed under BSD-3-Clause
 * Sonic library, authored by Bill Cox, Copyright (C) 2010 Bill Cox, licensed under Apache 2.0 license

diff --git a/april_api.h b/april_api.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2022 abb128
+ * Copyright (C) 2025 abb128
  * 
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

diff --git a/cmake/FindONNXRuntime.cmake b/cmake/FindONNXRuntime.cmake
diff --git a/download_onnx_linux_x64.sh b/download_onnx_linux_x64.sh
diff --git a/download_onnx_windows_x64.sh b/download_onnx_windows_x64.sh
diff --git a/example.cpp b/example.cpp
@@ -1,3 +1,6 @@
+// For basic live captioning of desktop audio, run it like so:
+// parec --format=s16 --rate=16000 --channels=1 --latency-ms=100 --device=@DEFAULT_MONITOR@ | ./main - /path/to/model.gguf
+
 #include <stdio.h>
 #include <cstdlib>
 #include <cstring>
@@ -128,8 +131,8 @@ int main(int argc, char *argv[]){
         // Reading stdin mode. It's assumed that the input data is pcm16 audio,
         // sampled in the model's sample rate.
         // You can achieve this on Linux like this:
-        // $ parec --format=s16 --rate=16000 --channels=1 --latency-ms=100 | ./main - /path/to/model.april
-
+        // $ parec --format=s16 --rate=16000 --channels=1 --latency-ms=100 --device=@DEFAULT_MONITOR@ | ./main - /path/to/model.gguf
+ 
         char data[BUFFER_SIZE];
         ssize_t r;
         for(;;){