diff --git a/common/arg.cpp b/common/arg.cpp index 40af7e574830f..85ba2182575cf 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3416,5 +3416,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--default-client-config"}, "JSON_FNAME", + string_format("JSON file containing the default client config"), + [](common_params & params, const std::string & value) { + params.public_default_client_config = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_DEFAULT_CLIENT_CONFIG")); + return ctx_arg; } diff --git a/common/common.h b/common/common.h index 8922090e7b10d..50c31cd5741a7 100644 --- a/common/common.h +++ b/common/common.h @@ -370,6 +370,7 @@ struct common_params { std::string hostname = "127.0.0.1"; std::string public_path = ""; // NOLINT + std::string public_default_client_config = ""; // NOLINT std::string chat_template = ""; // NOLINT bool use_jinja = false; // NOLINT bool enable_chat_template = true; diff --git a/tools/server/public/index.html.gz b/tools/server/public/index.html.gz index 53b71079c1e2a..0e3ee1e8b0336 100644 Binary files a/tools/server/public/index.html.gz and b/tools/server/public/index.html.gz differ diff --git a/tools/server/server.cpp b/tools/server/server.cpp index d3f6271931f62..a17492afc0f5b 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1905,6 +1906,7 @@ struct server_context { // slots / clients std::vector slots; json default_generation_settings_for_props; + json default_client_config = json::object(); server_queue queue_tasks; server_response queue_results; @@ -2097,6 +2099,15 @@ struct server_context { default_generation_settings_for_props = slots[0].to_json(); + if (!params_base.public_default_client_config.empty()) { + std::ifstream file(params_base.public_default_client_config); + LOG_INF("%s: Loading default client config from %s\n", __func__, params_base.public_default_client_config.c_str()); + if (!file.is_open()) { + throw std::runtime_error("Error: default client config file not open"); + } + file >> default_client_config; + } + // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used) { @@ -3851,6 +3862,11 @@ int main(int argc, char ** argv) { res_ok(res, health); }; + const auto handle_default_config = [&](const httplib::Request &, httplib::Response & res) { + // default client-side config + res_ok(res, ctx_server.default_client_config); + }; + const auto handle_slots = [&](const httplib::Request & req, httplib::Response & res) { if (!params.endpoint_slots) { res_error(res, format_error_response("This server does not support slots endpoint. Start it with `--slots`", ERROR_TYPE_NOT_SUPPORTED)); @@ -4830,6 +4846,7 @@ int main(int argc, char ** argv) { // register API routes svr->Get ("/health", handle_health); // public endpoint (no API key check) + svr->Get ("/defaultConfig.json", handle_default_config); // public endpoint (no API key check) svr->Get ("/metrics", handle_metrics); svr->Get ("/props", handle_props); svr->Post("/props", handle_props_change); diff --git a/tools/server/webui/src/components/SettingDialog.tsx b/tools/server/webui/src/components/SettingDialog.tsx index 45a8d73b00592..c0d8a29cf956e 100644 --- a/tools/server/webui/src/components/SettingDialog.tsx +++ b/tools/server/webui/src/components/SettingDialog.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useState, useEffect } from 'react'; import { useAppContext } from '../utils/app.context'; import { CONFIG_DEFAULT, CONFIG_INFO } from '../Config'; import { isDev } from '../Config'; @@ -285,6 +285,18 @@ export default function SettingDialog({ ); const { showConfirm, showAlert } = useModals(); + // get default client settings + useEffect(() => { + StorageUtils.setDefaultConfig().then((wasChanged: boolean) => { + if (wasChanged) { + console.log('Setting default config'); + const newConfig = StorageUtils.getConfig(); + saveConfig(newConfig); + setLocalConfig(JSON.parse(JSON.stringify(newConfig))); + } + }); + }, []); + const resetConfig = async () => { if (await showConfirm('Are you sure you want to reset all settings?')) { setLocalConfig(CONFIG_DEFAULT); diff --git a/tools/server/webui/src/utils/storage.ts b/tools/server/webui/src/utils/storage.ts index 505693e9272ac..cd0f69457e133 100644 --- a/tools/server/webui/src/utils/storage.ts +++ b/tools/server/webui/src/utils/storage.ts @@ -213,6 +213,23 @@ const StorageUtils = { localStorage.setItem('theme', theme); } }, + async setDefaultConfig(): Promise { + if (localStorage.getItem('config') === null) { + try { + const response = await fetch('/defaultConfig.json'); + const defaultConfig = await response.json(); + + // Ensure there still is no config when we overwrite it + if (localStorage.getItem('config') === null) { + localStorage.setItem('config', JSON.stringify(defaultConfig)); + } + return true; + } catch (e) { + console.error(e); + } + } + return false; + }, }; export default StorageUtils;