-
Notifications
You must be signed in to change notification settings - Fork 76
Expand file tree
/
Copy pathMakefile
More file actions
156 lines (136 loc) · 7.86 KB
/
Copy pathMakefile
File metadata and controls
156 lines (136 loc) · 7.86 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
ASSETS_DIR := internal/plugin/embed/assets
ORT_VERSION := 1.24.2
# ORT dropped Intel Mac support after v1.23.2; pin darwin/amd64 to the last working version.
ORT_VERSION_DARWIN_AMD64 := 1.23.2
MODEL_REPO := Xenova/bge-small-en-v1.5
HF_BASE := https://huggingface.co/$(MODEL_REPO)/resolve/main
ORT_BASE := https://github.com/microsoft/onnxruntime/releases/download/v$(ORT_VERSION)
.PHONY: fetch-assets fetch-model fetch-ort-libs clean-assets web css build test bench test-integration \
corpora check-filenames check-nul-bytes \
_ort-darwin-arm64 _ort-darwin-amd64 _ort-linux-amd64 _ort-linux-arm64 _ort-windows-amd64
## fetch-assets: download the model, tokenizer, and all platform ORT libraries.
fetch-assets: fetch-model fetch-ort-libs
## fetch-model: download model_int8.onnx and tokenizer.json from HuggingFace.
fetch-model:
@echo "==> Downloading bge-small-en-v1.5 INT8 model..."
@mkdir -p $(ASSETS_DIR)
@curl -fL --progress-bar \
"$(HF_BASE)/onnx/model_int8.onnx" \
-o "$(ASSETS_DIR)/model_int8.onnx"
@echo "==> Downloading tokenizer.json..."
@curl -fL --progress-bar \
"https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/tokenizer.json" \
-o "$(ASSETS_DIR)/tokenizer.json"
@echo " model_int8.onnx: $$(du -sh $(ASSETS_DIR)/model_int8.onnx | cut -f1)"
@echo " tokenizer.json: $$(du -sh $(ASSETS_DIR)/tokenizer.json | cut -f1)"
@echo "==> Model assets ready."
## fetch-ort-libs: download ORT native libraries for all supported platforms.
fetch-ort-libs:
@echo "==> Downloading ORT $(ORT_VERSION) native libraries..."
@mkdir -p $(ASSETS_DIR)
@$(MAKE) -s _ort-darwin-arm64
@$(MAKE) -s _ort-darwin-amd64
@$(MAKE) -s _ort-linux-amd64
@$(MAKE) -s _ort-linux-arm64
@$(MAKE) -s _ort-windows-amd64
@echo "==> ORT native libraries ready."
_ort-darwin-arm64:
@echo " Fetching darwin/arm64..."
@curl -fL --progress-bar \
"$(ORT_BASE)/onnxruntime-osx-arm64-$(ORT_VERSION).tgz" \
-o "/tmp/ort-osx-arm64.tgz"
@tar -xzf /tmp/ort-osx-arm64.tgz -C /tmp onnxruntime-osx-arm64-$(ORT_VERSION)/lib/libonnxruntime.dylib 2>/dev/null || \
tar -xzf /tmp/ort-osx-arm64.tgz -C /tmp --strip-components=2 --wildcards '*/lib/libonnxruntime.dylib'
@cp /tmp/onnxruntime-osx-arm64-$(ORT_VERSION)/lib/libonnxruntime.dylib $(ASSETS_DIR)/libonnxruntime_darwin_arm64.dylib 2>/dev/null || \
cp /tmp/libonnxruntime.dylib $(ASSETS_DIR)/libonnxruntime_darwin_arm64.dylib
@echo " darwin/arm64: $$(du -sh $(ASSETS_DIR)/libonnxruntime_darwin_arm64.dylib | cut -f1)"
_ort-darwin-amd64:
@echo " Fetching darwin/amd64 (ORT $(ORT_VERSION_DARWIN_AMD64) — last release with Intel Mac support)..."
@curl -fL --progress-bar \
"https://github.com/microsoft/onnxruntime/releases/download/v$(ORT_VERSION_DARWIN_AMD64)/onnxruntime-osx-x86_64-$(ORT_VERSION_DARWIN_AMD64).tgz" \
-o "/tmp/ort-osx-amd64.tgz"
@rm -rf /tmp/ort-osx-amd64-extract && mkdir -p /tmp/ort-osx-amd64-extract
@tar -xzf /tmp/ort-osx-amd64.tgz -C /tmp/ort-osx-amd64-extract/
@find /tmp/ort-osx-amd64-extract/ -name 'libonnxruntime*.dylib' | head -1 | xargs -I{} cp {} $(ASSETS_DIR)/libonnxruntime_darwin_amd64.dylib
@echo " darwin/amd64: $$(du -sh $(ASSETS_DIR)/libonnxruntime_darwin_amd64.dylib | cut -f1)"
_ort-linux-amd64:
@echo " Fetching linux/amd64..."
@curl -fL --progress-bar \
"$(ORT_BASE)/onnxruntime-linux-x64-$(ORT_VERSION).tgz" \
-o "/tmp/ort-linux-amd64.tgz"
@tar -xzf /tmp/ort-linux-amd64.tgz -C /tmp onnxruntime-linux-x64-$(ORT_VERSION)/lib/libonnxruntime.so.$(ORT_VERSION) 2>/dev/null || \
tar -xzf /tmp/ort-linux-amd64.tgz -C /tmp --strip-components=2 --wildcards '*/lib/libonnxruntime.so.*'
@cp /tmp/onnxruntime-linux-x64-$(ORT_VERSION)/lib/libonnxruntime.so.$(ORT_VERSION) $(ASSETS_DIR)/libonnxruntime_linux_amd64.so 2>/dev/null || \
find /tmp -name 'libonnxruntime.so.*' | head -1 | xargs -I{} cp {} $(ASSETS_DIR)/libonnxruntime_linux_amd64.so
@echo " linux/amd64: $$(du -sh $(ASSETS_DIR)/libonnxruntime_linux_amd64.so | cut -f1)"
_ort-linux-arm64:
@echo " Fetching linux/arm64..."
@curl -fL --progress-bar \
"$(ORT_BASE)/onnxruntime-linux-aarch64-$(ORT_VERSION).tgz" \
-o "/tmp/ort-linux-arm64.tgz"
@tar -xzf /tmp/ort-linux-arm64.tgz -C /tmp onnxruntime-linux-aarch64-$(ORT_VERSION)/lib/libonnxruntime.so.$(ORT_VERSION) 2>/dev/null || \
tar -xzf /tmp/ort-linux-arm64.tgz -C /tmp --strip-components=2 --wildcards '*/lib/libonnxruntime.so.*'
@cp /tmp/onnxruntime-linux-aarch64-$(ORT_VERSION)/lib/libonnxruntime.so.$(ORT_VERSION) $(ASSETS_DIR)/libonnxruntime_linux_arm64.so 2>/dev/null || \
find /tmp -name 'libonnxruntime.so.*' | head -1 | xargs -I{} cp {} $(ASSETS_DIR)/libonnxruntime_linux_arm64.so
@echo " linux/arm64: $$(du -sh $(ASSETS_DIR)/libonnxruntime_linux_arm64.so | cut -f1)"
_ort-windows-amd64:
@echo " Fetching windows/amd64..."
@curl -fL --progress-bar \
"$(ORT_BASE)/onnxruntime-win-x64-$(ORT_VERSION).zip" \
-o "/tmp/ort-win-x64.zip"
@rm -rf /tmp/ort-win-x64-extract && mkdir -p /tmp/ort-win-x64-extract
@unzip -q /tmp/ort-win-x64.zip -d /tmp/ort-win-x64-extract
@find /tmp/ort-win-x64-extract -name 'onnxruntime.dll' | head -1 | xargs -I{} cp {} $(ASSETS_DIR)/onnxruntime_windows_amd64.dll
@echo " windows/amd64: $$(du -sh $(ASSETS_DIR)/onnxruntime_windows_amd64.dll | cut -f1)"
## clean-assets: remove downloaded binary assets.
clean-assets:
@echo "==> Removing downloaded assets..."
@rm -f $(ASSETS_DIR)/model_int8.onnx
@rm -f $(ASSETS_DIR)/tokenizer.json
@rm -f $(ASSETS_DIR)/libonnxruntime_*.dylib
@rm -f $(ASSETS_DIR)/libonnxruntime_*.so
@rm -f $(ASSETS_DIR)/onnxruntime_*.dll
@echo "==> Done."
## web: compile Tailwind CSS via Vite (requires Node.js + npm).
web:
@cd web && npm ci --silent && npm run build --silent
## css: rebuild only the CSS bundle (faster than full web target; skips npm ci).
css:
@cd web && npm run build --silent
## build: build the server binary (requires fetch-assets and web first).
build: web
@go build -tags localassets -o muninndb-server ./cmd/muninn/...
## test: run unit tests across all packages.
test:
@go test ./...
## bench: run E2E benchmark suite.
bench:
go test -bench=BenchmarkE2E -benchmem -benchtime=3s ./internal/bench/...
## test-integration: run integration tests (requires no muninn already running on :8750).
## Builds the binary, exercises the full start/stop/restart lifecycle, then cleans up.
## If :8750 is busy the suite SKIPs visibly; MUNINN_REQUIRE_INTEGRATION=1 makes that a
## failure instead, which is what CI sets (#812).
test-integration:
@go test -tags integration -v -timeout 120s ./cmd/muninn/...
## corpora: run the standing measurement corpora and write .artifacts/corpora.txt.
## The four harnesses in internal/engine/activation, normalised so two runs of an
## unchanged tree are byte-identical and a diff shows only moved numbers. They also run
## inside `go test ./...`; this target exists to produce the diffable artifact.
corpora:
@bash scripts/run-corpora.sh
## check-filenames: fail if a source file's NAME silently excludes it from the build (#814).
check-filenames:
@bash scripts/check-filename-build-constraints.sh
## check-nul-bytes: fail if a tracked source file contains a NUL byte, which git renders as
## a binary blob invisible to git diff and every diff-based review surface (#827).
check-nul-bytes:
@bash scripts/check-nul-bytes.sh
# The eval-bible-* targets were removed here: they build ./cmd/eval-bible/... and run
# scripts/eval-bible-setup.sh, neither of which is in the public repo (cmd/eval*/ and
# scripts/eval-* are gitignored as internal development tooling). Advertising targets that
# cannot run from a fresh clone is worse than not listing them. If the retrieval-quality
# eval harness is published later, restore them alongside it.
#
# testdata/bible/ went with them (it held only a .gitkeep). An empty data directory whose
# only loader is gitignored reads as "there is a corpus here"; there was not one, and it
# was cited as if it were the standing corpora. The standing corpora are `make corpora`.