diff --git a/Cargo.lock b/Cargo.lock index 8a72edd..b652ea1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.21" @@ -298,6 +307,19 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.5.53" @@ -437,9 +459,17 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "cydonia" +version = "0.0.9" +dependencies = [ + "cydonia-core", + "cydonia-deepseek", +] + [[package]] name = "cydonia-candle" -version = "0.0.0" +version = "0.0.9" dependencies = [ "anyhow", "candle-core", @@ -453,9 +483,57 @@ dependencies = [ "tracing", ] +[[package]] +name = "cydonia-cli" +version = "0.0.9" +dependencies = [ + "anyhow", + "chrono", + "clap", + "cydonia", + "dirs", + "futures-util", + "schemars", + "serde", + "serde_json", + "tokio", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "cydonia-core" +version = "0.0.9" +dependencies = [ + "anyhow", + "async-stream", + "derive_more", + "futures-core", + "futures-util", + "reqwest", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "cydonia-deepseek" +version = "0.0.9" +dependencies = [ + "anyhow", + "async-stream", + "cydonia-core", + "futures-core", + "futures-util", + "serde", + "serde_json", + "tracing", +] + [[package]] name = "cydonia-model" -version = "0.0.0" +version = "0.0.9" dependencies = [ "anyhow", "serde", @@ -1300,6 +1378,30 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -2947,58 +3049,6 @@ dependencies = [ "yoke 0.7.5", ] -[[package]] -name = "ullm" -version = "0.0.9" -dependencies = [ - "ullm-core", - "ullm-deepseek", -] - -[[package]] -name = "ullm-cli" -version = "0.0.9" -dependencies = [ - "anyhow", - "clap", - "dirs", - "futures-util", - "serde", - "toml", - "tracing", - "tracing-subscriber", - "ullm", -] - -[[package]] -name = "ullm-core" -version = "0.0.9" -dependencies = [ - "anyhow", - "async-stream", - "derive_more", - "futures-core", - "futures-util", - "reqwest", - "schemars", - "serde", - "serde_json", -] - -[[package]] -name = "ullm-deepseek" -version = "0.0.9" -dependencies = [ - "anyhow", - "async-stream", - "futures-core", - "futures-util", - "serde", - "serde_json", - "tracing", - "ullm-core", -] - [[package]] name = "unicode-ident" version = "1.0.22" @@ -3280,6 +3330,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index c7101e2..dee1ba7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,21 +6,21 @@ members = ["crates/*", "llm/*", "legacy/*"] version = "0.0.9" edition = "2024" authors = ["clearloop "] -license = "MIT" +license = "GPL-3.0" repository = "https://github.com/clearloop/cydonia" -documentation = "https://cydonia.docs.rs" keywords = ["llm", "agent", "ai"] [workspace.dependencies] -candle = { path = "crates/candle", package = "cydonia-candle" } -deepseek = { path = "llm/deepseek", package = "ullm-deepseek" } -model = { path = "legacy/model", package = "cydonia-model" } -ullm = { path = "crates/ullm" } -ucore = { path = "crates/core", package = "ullm-core" } -ucli = { path = "crates/cli", package = "ullm-cli" } +candle = { path = "crates/candle", package = "cydonia-candle", version = "0.0.9" } +cli = { path = "crates/cli", package = "cydonia-cli", version = "0.0.9" } +deepseek = { path = "llm/deepseek", package = "cydonia-deepseek", version = "0.0.9" } +model = { path = "legacy/model", package = "cydonia-model", version = "0.0.9" } +cydonia = { path = "crates/cydonia", version = "0.0.9" } +ccore = { path = "crates/core", package = "cydonia-core", version = "0.0.9" } # crates.io anyhow = "1" +chrono = "0.4" async-stream = "0.3" bytes = "1.11.0" clap = { version = "4.5", features = ["derive"] } @@ -47,3 +47,6 @@ llamac-sys = { version = "0.1.86", package = "llama-cpp-sys-2" } once_cell = "1.21" rand = "0.9.2" tokenizers = "0.21.0" + +[workspace.metadata.conta] +packages = ["ccore", "deepseek", "cydonia", "cli"] diff --git a/README.md b/README.md index 210c98f..f3e6783 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,35 @@ # Cydonia -The Agent framework. +Unified LLM Interface - A Rust framework for building LLM-powered agents. + +## Features + +- Unified interface for multiple LLM providers +- Streaming support with thinking mode +- Tool calling with automatic argument accumulation +- Agent framework for building autonomous assistants + +## Crates + +| Crate | Description | +|-------|-------------| +| `cydonia` | Umbrella crate re-exporting all components | +| `cydonia-core` | Core abstractions (LLM, Agent, Chat, Message) | +| `cydonia-deepseek` | DeepSeek provider implementation | +| `cydonia-cli` | Command line interface | + +## Quick Start + +```rust +use cydonia::{Chat, DeepSeek, LLM, Message}; + +let client = reqwest::Client::new(); +let provider = DeepSeek::new(client, "your-api-key")?; +let mut chat = provider.chat(config); + +let response = chat.send(Message::user("Hello!")).await?; +``` + +## License + +MIT diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index cf55045..a1cc846 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -1,22 +1,27 @@ [package] -name = "ullm-cli" +name = "cydonia-cli" +description = "Cydonia command line interfaces" +documentation = "https://docs.rs/cydonia-cli" version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true repository.workspace = true -documentation.workspace = true keywords.workspace = true [dependencies] -ullm.workspace = true +cydonia.workspace = true # crates-io dependencies anyhow.workspace = true +chrono.workspace = true clap.workspace = true dirs.workspace = true futures-util.workspace = true serde.workspace = true toml.workspace = true +schemars.workspace = true +serde_json.workspace = true +tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/crates/cli/README.md b/crates/cli/README.md new file mode 100644 index 0000000..64ceae3 --- /dev/null +++ b/crates/cli/README.md @@ -0,0 +1,19 @@ +# cydonia-cli + +Command line interface for cydonia. + +## Overview + +Provides CLI tools for interacting with LLMs, including: + +- Chat interface with streaming support +- Agent framework with tool calling +- Configuration management + +## Agents + +- `Anto` - Test agent with `get_time` tool for verifying tool calling + +## License + +MIT diff --git a/crates/cli/bin/ullm.rs b/crates/cli/bin/cydonia.rs similarity index 87% rename from crates/cli/bin/ullm.rs rename to crates/cli/bin/cydonia.rs index a0f06e2..2a51b25 100644 --- a/crates/cli/bin/ullm.rs +++ b/crates/cli/bin/cydonia.rs @@ -1,6 +1,6 @@ use anyhow::Result; use clap::Parser; -use ucli::{App, Command, Config}; +use cydonia_cli::{App, Command, Config}; #[tokio::main] async fn main() -> Result<()> { diff --git a/crates/cli/src/agents/anto.rs b/crates/cli/src/agents/anto.rs new file mode 100644 index 0000000..de80bb0 --- /dev/null +++ b/crates/cli/src/agents/anto.rs @@ -0,0 +1,60 @@ +//! Anto agent - a basic agent to verify tool calling + +use anyhow::Result; +use chrono::Utc; +use cydonia::{Agent, Message, StreamChunk, Tool, ToolCall}; +use schemars::JsonSchema; +use serde::Deserialize; + +/// Anto - a basic agent with tools for testing tool calls +#[derive(Clone)] +pub struct Anto; + +/// Parameters for the get_time tool +#[allow(dead_code)] +#[derive(JsonSchema, Deserialize)] +struct GetTimeParams { + /// If returns UNIX timestamp instead + timestamp: bool, +} + +impl Agent for Anto { + type Chunk = StreamChunk; + + const SYSTEM_PROMPT: &str = "You are Anto, a helpful assistant. You can get the current time."; + + fn tools() -> Vec { + vec![Tool { + name: "get_time".into(), + description: "Gets the current UTC time in ISO 8601 format.".into(), + parameters: schemars::schema_for!(GetTimeParams), + strict: true, + }] + } + + async fn dispatch(&self, tools: &[ToolCall]) -> Vec { + tools + .iter() + .map(|call| { + let result = match call.function.name.as_str() { + "get_time" => { + tracing::debug!("get_time arguments: {}", call.function.arguments); + let args = serde_json::from_str::(&call.function.arguments) + .unwrap(); + if args.timestamp { + Utc::now().timestamp().to_string() + } else { + Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string() + } + } + _ => format!("Unknown tool: {}", call.function.name), + }; + Message::tool(result, call.id.clone()) + }) + .collect() + } + + async fn chunk(&self, chunk: &StreamChunk) -> Result { + Ok(chunk.clone()) + } +} diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs new file mode 100644 index 0000000..0418cd5 --- /dev/null +++ b/crates/cli/src/agents/mod.rs @@ -0,0 +1,13 @@ +//! CLI Agents + +pub use anto::Anto; +use clap::ValueEnum; + +mod anto; + +/// Available agent types +#[derive(Debug, Clone, Copy, ValueEnum)] +pub enum AgentKind { + /// Anto - basic agent with echo tool for testing + Anto, +} diff --git a/crates/cli/src/chat.rs b/crates/cli/src/chat.rs index df02820..51e92ba 100644 --- a/crates/cli/src/chat.rs +++ b/crates/cli/src/chat.rs @@ -1,15 +1,15 @@ //! Chat command use super::Config; +use crate::agents::{AgentKind, Anto}; use anyhow::Result; use clap::{Args, ValueEnum}; +use cydonia::{Agent, Chat, Client, DeepSeek, LLM, Message, StreamChunk}; use futures_util::StreamExt; use std::{ fmt::{Display, Formatter}, io::{BufRead, Write}, }; -use ullm::DeepSeek; -use ullm::{Chat, Client, LLM, Message}; /// Chat command arguments #[derive(Debug, Args)] @@ -18,6 +18,14 @@ pub struct ChatCmd { #[arg(short, long, default_value = "deepseek")] pub model: Model, + /// The agent to use for the chat + #[arg(short, long)] + pub agent: Option, + + /// Whether to enable thinking + #[arg(short, long)] + pub think: bool, + /// The message to send (if empty, starts interactive mode) pub message: Option, } @@ -25,7 +33,7 @@ pub struct ChatCmd { impl ChatCmd { /// Run the chat command pub async fn run(&self, stream: bool) -> Result<()> { - let config = Config::load()?; + let mut config = Config::load()?; let key = config .key .get(&self.model.to_string()) @@ -34,9 +42,28 @@ impl ChatCmd { Model::Deepseek => DeepSeek::new(Client::new(), key)?, }; - let mut chat = provider.chat(config.config().clone()); + // override the think flag in the config + config.config.think = self.think; + + // run the chat + match self.agent { + Some(AgentKind::Anto) => { + let mut chat = provider.chat(config.config().clone()).system(Anto); + self.run_chat(&mut chat, stream).await + } + None => { + let mut chat = provider.chat(config.config().clone()); + self.run_chat(&mut chat, stream).await + } + } + } + + async fn run_chat(&self, chat: &mut Chat, stream: bool) -> Result<()> + where + A: Agent, + { if let Some(msg) = &self.message { - Self::send(&mut chat, Message::user(msg), stream).await?; + Self::send(chat, Message::user(msg), stream).await?; } else { let stdin = std::io::stdin(); let mut stdout = std::io::stdout(); @@ -57,54 +84,50 @@ impl ChatCmd { break; } - Self::send(&mut chat, Message::user(input), stream).await?; + Self::send(chat, Message::user(input), stream).await?; } } Ok(()) } - async fn send(chat: &mut Chat, message: Message, stream: bool) -> Result<()> { + async fn send(chat: &mut Chat, message: Message, stream: bool) -> Result<()> + where + A: Agent, + { if stream { let mut response_content = String::new(); - { - let mut reasoning = false; - let mut stream = std::pin::pin!(chat.stream(message)); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - if let Some(content) = chunk.content() { - if reasoning { - print!("\ncontent: "); - reasoning = false; - } - print!("{content}"); - response_content.push_str(content); + let mut reasoning = false; + let mut stream = std::pin::pin!(chat.stream(message)); + while let Some(Ok(chunk)) = stream.next().await { + if let Some(content) = chunk.content() { + if reasoning { + println!("\n\n\nCONTENT"); + reasoning = false; } + print!("{content}"); + response_content.push_str(content); + } - if let Some(reasoning_content) = chunk.reasoning_content() { - if !reasoning { - print!("thinking: "); - reasoning = true; - } - print!("{reasoning_content}"); - response_content.push_str(reasoning_content); + if let Some(reasoning_content) = chunk.reasoning_content() { + if !reasoning { + println!("REASONING"); + reasoning = true; } + print!("{reasoning_content}"); + response_content.push_str(reasoning_content); } } println!(); - chat.messages - .push(Message::assistant(&response_content).into()); } else { let response = chat.send(message).await?; if let Some(reasoning_content) = response.reasoning() { - println!("reasoning: {reasoning_content}"); + println!("REASONING\n{reasoning_content}"); } if let Some(content) = response.message() { - println!("{content}"); + println!("\n\nCONTENT\n{content}"); } - chat.messages - .push(Message::assistant(response.message().unwrap_or(&String::new())).into()); } Ok(()) } diff --git a/crates/cli/src/config.rs b/crates/cli/src/config.rs index 6916820..3880276 100644 --- a/crates/cli/src/config.rs +++ b/crates/cli/src/config.rs @@ -10,7 +10,7 @@ static CONFIG: LazyLock = #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Config { /// The configuration for the CLI - config: ullm::General, + pub config: cydonia::General, /// The API keys for LLMs pub key: BTreeMap, @@ -31,7 +31,7 @@ impl Config { } /// Get the core config - pub fn config(&self) -> &ullm::General { + pub fn config(&self) -> &cydonia::General { &self.config } } @@ -39,7 +39,7 @@ impl Config { impl Default for Config { fn default() -> Self { Self { - config: ullm::General::default(), + config: cydonia::General::default(), key: [("deepseek".to_string(), "YOUR_API_KEY".to_string())] .into_iter() .collect::<_>(), diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 5fcd4df..8ab3a79 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,12 +1,13 @@ //! CLI commands for ullm -mod chat; -mod config; - use clap::{Parser, Subcommand}; use tracing_subscriber::{EnvFilter, fmt}; pub use {chat::ChatCmd, config::Config}; +mod agents; +mod chat; +mod config; + /// Unified LLM Interface CLI #[derive(Debug, Parser)] #[command(name = "ullm", version, about)] diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 52fb1e1..53e61a2 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,11 +1,12 @@ [package] -name = "ullm-core" +name = "cydonia-core" +description = "Cydonia core abstractions" +documentation = "https://docs.rs/cydonia-core" version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true repository.workspace = true -documentation.workspace = true keywords.workspace = true [dependencies] diff --git a/crates/core/README.md b/crates/core/README.md new file mode 100644 index 0000000..3f7ffac --- /dev/null +++ b/crates/core/README.md @@ -0,0 +1,18 @@ +# cydonia-core + +Core abstractions for the Unified LLM Interface. + +## Overview + +This crate provides the foundational types and traits for building LLM applications: + +- `LLM` - Provider trait for LLM backends +- `Agent` - Trait for building tool-using agents +- `Chat` - Chat session management with streaming support +- `Message` - Chat message types (user, assistant, system, tool) +- `Tool` / `ToolCall` - Function calling abstractions +- `StreamChunk` - Streaming response handling + +## License + +MIT diff --git a/crates/core/src/agent.rs b/crates/core/src/agent.rs index e7d52e7..f4910a8 100644 --- a/crates/core/src/agent.rs +++ b/crates/core/src/agent.rs @@ -1,6 +1,6 @@ //! Turbofish Agent library -use crate::{Message, StreamChunk, Tool, ToolCall, ToolChoice, message::ToolMessage}; +use crate::{Message, StreamChunk, Tool, ToolCall, ToolChoice}; use anyhow::Result; /// A trait for turbofish agents @@ -14,7 +14,9 @@ pub trait Agent: Clone { const SYSTEM_PROMPT: &str; /// The tools for the agent - const TOOLS: Vec = Vec::new(); + fn tools() -> Vec { + Vec::new() + } /// Filter the messages to match required tools for the agent fn filter(&self, _message: &str) -> ToolChoice { @@ -22,16 +24,15 @@ pub trait Agent: Clone { } /// Dispatch tool calls - fn dispatch(&self, tools: &[ToolCall]) -> impl Future> { + fn dispatch(&self, tools: &[ToolCall]) -> impl Future> { async move { tools .iter() - .map(|tool| ToolMessage { - tool: tool.id.clone(), - message: Message::tool(format!( - "function {} not available", - tool.function.name - )), + .map(|tool| { + Message::tool( + format!("function {} not available", tool.function.name), + tool.id.clone(), + ) }) .collect() } diff --git a/crates/core/src/chat.rs b/crates/core/src/chat.rs index 1f1e4d7..1de54d5 100644 --- a/crates/core/src/chat.rs +++ b/crates/core/src/chat.rs @@ -1,13 +1,12 @@ //! Chat abstractions for the unified LLM Interfaces use crate::{ - Agent, Config, FinishReason, General, LLM, Response, Role, - message::{AssistantMessage, Message, ToolMessage}, + Agent, Config, FinishReason, General, LLM, Response, Role, ToolCall, message::Message, }; use anyhow::Result; use futures_core::Stream; use futures_util::StreamExt; -use serde::Serialize; +use std::collections::HashMap; const MAX_TOOL_CALLS: usize = 16; @@ -18,7 +17,7 @@ pub struct Chat { pub config: P::ChatConfig, /// Chat history in memory - pub messages: Vec, + pub messages: Vec, /// The LLM provider provider: P, @@ -44,21 +43,35 @@ impl Chat { } impl Chat { + /// Get the chat messages for API requests. + pub fn messages(&self) -> Vec { + self.messages + .clone() + .into_iter() + .map(|mut m| { + if m.tool_calls.is_empty() { + m.reasoning_content = String::new(); + } + m + }) + .collect() + } + /// Add the system prompt to the chat pub fn system(mut self, agent: B) -> Chat { let mut messages = self.messages; if messages.is_empty() { - messages.push(Message::system(A::SYSTEM_PROMPT).into()); - } else if let Some(ChatMessage::System(_)) = messages.first() { - messages.insert(0, Message::system(A::SYSTEM_PROMPT).into()); + messages.push(Message::system(B::SYSTEM_PROMPT)); + } else if messages.first().map(|m| m.role) == Some(Role::System) { + messages.insert(0, Message::system(B::SYSTEM_PROMPT)); } else { - messages = vec![Message::system(A::SYSTEM_PROMPT).into()] + messages = vec![Message::system(B::SYSTEM_PROMPT)] .into_iter() .chain(messages) .collect(); } - self.config = self.config.with_tools(A::TOOLS); + self.config = self.config.with_tools(B::tools()); Chat { messages, provider: self.provider, @@ -73,16 +86,23 @@ impl Chat { let config = self .config .with_tool_choice(self.agent.filter(message.content.as_str())); - self.messages.push(message.into()); - + self.messages.push(message); for _ in 0..MAX_TOOL_CALLS { - let response = self.provider.send(&config, &self.messages).await?; + let response = self.provider.send(&config, &self.messages()).await?; + if let Some(message) = response.message() { + self.messages.push(Message::assistant( + message, + response.reasoning().cloned(), + response.tool_calls(), + )); + } + let Some(tool_calls) = response.tool_calls() else { return Ok(response); }; let result = self.agent.dispatch(tool_calls).await; - self.messages.extend(result.into_iter().map(Into::into)); + self.messages.extend(result); } anyhow::bail!("max tool calls reached"); @@ -96,26 +116,43 @@ impl Chat { let config = self .config .with_tool_choice(self.agent.filter(message.content.as_str())); - self.messages.push(message.into()); + self.messages.push(message); async_stream::try_stream! { for _ in 0..MAX_TOOL_CALLS { - let messages = self.messages.clone(); + let messages = self.messages(); let inner = self.provider.stream(config.clone(), &messages, self.usage); futures_util::pin_mut!(inner); - let mut tool_calls = None; + let mut tool_calls: HashMap = HashMap::new(); let mut message = String::new(); + let mut reasoning = String::new(); while let Some(chunk) = inner.next().await { let chunk = chunk?; if let Some(calls) = chunk.tool_calls() { - tool_calls = Some(calls.to_vec()); + for call in calls { + let entry = tool_calls.entry(call.index).or_default(); + if !call.id.is_empty() { + entry.id.clone_from(&call.id); + } + if !call.call_type.is_empty() { + entry.call_type.clone_from(&call.call_type); + } + if !call.function.name.is_empty() { + entry.function.name.clone_from(&call.function.name); + } + entry.function.arguments.push_str(&call.function.arguments); + } } if let Some(content) = chunk.content() { message.push_str(content); } + if let Some(reason) = chunk.reasoning_content() { + reasoning.push_str(reason); + } + yield self.agent.chunk(&chunk).await?; if let Some(reason) = chunk.reason() { match reason { @@ -126,60 +163,18 @@ impl Chat { } } - if !message.is_empty() { - self.messages.push(Message::assistant(&message).into()); - } - - if let Some(calls) = tool_calls { - let result = self.agent.dispatch(&calls).await; - self.messages.extend(result.into_iter().map(Into::into)); - } else { + let reasoning = if reasoning.is_empty() { None } else { Some(reasoning) }; + if tool_calls.is_empty() { + self.messages.push(Message::assistant(&message, reasoning, None)); break; + } else { + let mut calls: Vec<_> = tool_calls.into_values().collect(); + calls.sort_by_key(|c| c.index); + self.messages.push(Message::assistant(&message, reasoning, Some(&calls))); + let result = self.agent.dispatch(&calls).await; + self.messages.extend(result); } } - - Err(anyhow::anyhow!("max tool calls reached"))?; - } - } -} - -/// A chat message in memory -#[derive(Debug, Clone, Serialize)] -#[serde(untagged)] -pub enum ChatMessage { - /// A user message - User(Message), - - /// An assistant message - Assistant(AssistantMessage), - - /// A tool message - Tool(ToolMessage), - - /// A system message - System(Message), -} - -impl From for ChatMessage { - fn from(message: Message) -> Self { - match message.role { - Role::User => ChatMessage::User(message), - Role::Assistant => ChatMessage::Assistant(AssistantMessage { - message, - prefix: false, - reasoning: String::new(), - }), - Role::System => ChatMessage::System(message), - Role::Tool => ChatMessage::Tool(ToolMessage { - tool: String::new(), - message, - }), } } } - -impl From for ChatMessage { - fn from(message: ToolMessage) -> Self { - ChatMessage::Tool(message) - } -} diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 062c6ef..90d8c1a 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -20,8 +20,10 @@ pub struct General { /// The model to use pub model: String, + /// Whether to enable thinking + pub think: bool, + /// The tools to use - #[serde(skip_serializing_if = "Option::is_none")] pub tools: Option>, /// Whether to return the usage information in stream mode @@ -33,6 +35,7 @@ impl General { pub fn new(model: impl Into) -> Self { Self { model: model.into(), + think: false, tools: None, usage: false, } @@ -43,6 +46,7 @@ impl Default for General { fn default() -> Self { Self { model: "deepseek-chat".into(), + think: false, tools: None, usage: false, } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 17b1424..eb6f609 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -2,7 +2,7 @@ pub use { agent::Agent, - chat::{Chat, ChatMessage}, + chat::Chat, config::{Config, General}, message::{Message, Role}, provider::LLM, diff --git a/crates/core/src/message.rs b/crates/core/src/message.rs index 85e9f13..eb0f854 100644 --- a/crates/core/src/message.rs +++ b/crates/core/src/message.rs @@ -1,18 +1,37 @@ //! Turbofish LLM message +use crate::ToolCall; use serde::{Deserialize, Serialize}; /// A message in the chat -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize, Default)] pub struct Message { + /// The role of the message + pub role: Role, + /// The content of the message + #[serde(skip_serializing_if = "String::is_empty")] pub content: String, /// The name of the message + #[serde(skip_serializing_if = "String::is_empty")] pub name: String, - /// The role of the message - pub role: Role, + /// Whether to prefix the message + #[serde(skip_serializing_if = "Option::is_none")] + pub prefix: Option, + + /// The reasoning content + #[serde(skip_serializing_if = "String::is_empty")] + pub reasoning_content: String, + + /// The tool call id + #[serde(skip_serializing_if = "String::is_empty")] + pub tool_call_id: String, + + /// The tool calls + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tool_calls: Vec, } impl Message { @@ -20,8 +39,8 @@ impl Message { pub fn system(content: impl Into) -> Self { Self { role: Role::System, - name: String::new(), content: content.into(), + ..Default::default() } } @@ -29,62 +48,43 @@ impl Message { pub fn user(content: impl Into) -> Self { Self { role: Role::User, - name: String::new(), content: content.into(), + ..Default::default() } } /// Create a new assistant message - pub fn assistant(content: impl Into) -> Self { + pub fn assistant( + content: impl Into, + reasoning: Option, + tool_calls: Option<&[ToolCall]>, + ) -> Self { Self { role: Role::Assistant, - name: String::new(), content: content.into(), + reasoning_content: reasoning.unwrap_or_default(), + tool_calls: tool_calls.unwrap_or_default().to_vec(), + ..Default::default() } } /// Create a new tool message - pub fn tool(content: impl Into) -> Self { + pub fn tool(content: impl Into, call: impl Into) -> Self { Self { role: Role::Tool, - name: String::new(), content: content.into(), + tool_call_id: call.into(), + ..Default::default() } } } -/// A tool message in the chat -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ToolMessage { - /// The message - #[serde(flatten)] - pub message: Message, - - /// The tool call id - #[serde(alias = "tool_call_id")] - pub tool: String, -} - -/// An assistant message in the chat -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct AssistantMessage { - /// The message - #[serde(flatten)] - pub message: Message, - - /// Whether to prefix the message - pub prefix: bool, - - /// The reasoning content - #[serde(alias = "reasoning_content")] - pub reasoning: String, -} - /// The role of a message -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize, Default)] pub enum Role { /// The user role #[serde(rename = "user")] + #[default] User, /// The assistant role #[serde(rename = "assistant")] diff --git a/crates/core/src/provider.rs b/crates/core/src/provider.rs index b2f5010..65f70d6 100644 --- a/crates/core/src/provider.rs +++ b/crates/core/src/provider.rs @@ -1,6 +1,6 @@ //! Provider abstractions for the unified LLM Interfaces -use crate::{Chat, ChatMessage, Config, General, Response, StreamChunk}; +use crate::{Chat, Config, General, Message, Response, StreamChunk}; use anyhow::Result; use futures_core::Stream; use reqwest::Client; @@ -24,14 +24,14 @@ pub trait LLM: Sized + Clone { fn send( &mut self, config: &Self::ChatConfig, - messages: &[ChatMessage], + messages: &[Message], ) -> impl Future>; /// Send a message to the LLM with streaming fn stream( &mut self, config: Self::ChatConfig, - messages: &[ChatMessage], + messages: &[Message], usage: bool, ) -> impl Stream>; } diff --git a/crates/core/src/stream.rs b/crates/core/src/stream.rs index 559b5b0..be602d2 100644 --- a/crates/core/src/stream.rs +++ b/crates/core/src/stream.rs @@ -33,14 +33,16 @@ impl StreamChunk { pub fn content(&self) -> Option<&str> { self.choices .first() - .and_then(|choice| choice.delta.content.as_deref()) + .and_then(|c| c.delta.content.as_deref()) + .filter(|s| !s.is_empty()) } /// Get the reasoning content of the first choice pub fn reasoning_content(&self) -> Option<&str> { self.choices .first() - .and_then(|choice| choice.delta.reasoning_content.as_deref()) + .and_then(|c| c.delta.reasoning_content.as_deref()) + .filter(|s| !s.is_empty()) } /// Get the tool calls of the first choice diff --git a/crates/core/src/tool.rs b/crates/core/src/tool.rs index 96a46fa..25e015a 100644 --- a/crates/core/src/tool.rs +++ b/crates/core/src/tool.rs @@ -20,13 +20,18 @@ pub struct Tool { } /// A tool call made by the model -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct ToolCall { /// The ID of the tool call + #[serde(default, skip_serializing_if = "String::is_empty")] pub id: String, + /// The index of the tool call (used in streaming) + #[serde(default, skip_serializing)] + pub index: u32, + /// The type of tool (currently only "function") - #[serde(rename = "type")] + #[serde(default, rename = "type")] pub call_type: String, /// The function to call @@ -34,12 +39,14 @@ pub struct ToolCall { } /// A function call within a tool call -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct FunctionCall { /// The name of the function to call + #[serde(default, skip_serializing_if = "String::is_empty")] pub name: String, /// The arguments to pass to the function (JSON string) + #[serde(skip_serializing_if = "String::is_empty")] pub arguments: String, } diff --git a/crates/core/tests/response.rs b/crates/core/tests/response.rs index 607b42f..15b9cff 100644 --- a/crates/core/tests/response.rs +++ b/crates/core/tests/response.rs @@ -1,6 +1,6 @@ //! Tests for the response module -use ullm_core::{Response, StreamChunk}; +use cydonia_core::{Response, StreamChunk}; const DEEPSEEK_RESPONSE_JSON: &str = include_str!("../templates/deepseek/response.json"); const DEEPSEEK_STREAM_CHUNK_JSON: &str = include_str!("../templates/deepseek/stream.json"); diff --git a/crates/ullm/Cargo.toml b/crates/cydonia/Cargo.toml similarity index 62% rename from crates/ullm/Cargo.toml rename to crates/cydonia/Cargo.toml index 5dfdaa5..73bfc64 100644 --- a/crates/ullm/Cargo.toml +++ b/crates/cydonia/Cargo.toml @@ -1,13 +1,14 @@ [package] -name = "ullm" +name = "cydonia" +description = "Cydonia umbrella crate" +documentation = "https://docs.rs/cydonia" version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true repository.workspace = true -documentation.workspace = true keywords.workspace = true [dependencies] deepseek.workspace = true -ucore.workspace = true +ccore.workspace = true diff --git a/crates/cydonia/README.md b/crates/cydonia/README.md new file mode 100644 index 0000000..81c9afb --- /dev/null +++ b/crates/cydonia/README.md @@ -0,0 +1,19 @@ +# cydonia + +Unified LLM Interface - the umbrella crate. + +## Overview + +This crate re-exports all cydonia components for convenient usage: + +```rust +use cydonia::{Agent, Chat, DeepSeek, LLM, Message}; +``` + +## Supported Providers + +- DeepSeek (`deepseek-chat`, `deepseek-reasoner`) + +## License + +MIT diff --git a/crates/ullm/src/lib.rs b/crates/cydonia/src/lib.rs similarity index 53% rename from crates/ullm/src/lib.rs rename to crates/cydonia/src/lib.rs index a02ea39..1410e62 100644 --- a/crates/ullm/src/lib.rs +++ b/crates/cydonia/src/lib.rs @@ -2,5 +2,7 @@ //! //! This is the umbrella crate that re-exports all ullm components. +pub use ccore::{ + self, Agent, Chat, Client, Config, General, LLM, Message, StreamChunk, Tool, ToolCall, +}; pub use deepseek::DeepSeek; -pub use ucore::{self, Chat, ChatMessage, Client, Config, General, LLM, Message}; diff --git a/legacy/candle/Cargo.toml b/legacy/candle/Cargo.toml index 8139948..dbc4811 100644 --- a/legacy/candle/Cargo.toml +++ b/legacy/candle/Cargo.toml @@ -1,8 +1,13 @@ [package] name = "cydonia-candle" -version = "0.0.0" -edition = "2021" description = "Cydonia candle utils re-exports" +documentation = "https://docs.rs/cydonia-candle" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true [dependencies] anyhow.workspace = true diff --git a/legacy/candle/src/device.rs b/legacy/candle/src/device.rs index 5183244..5b0cb01 100644 --- a/legacy/candle/src/device.rs +++ b/legacy/candle/src/device.rs @@ -1,6 +1,6 @@ //! Device detection -use candle_core::{utils, Device}; +use candle_core::{Device, utils}; /// Detect the device pub fn detect(cpu: bool) -> anyhow::Result { diff --git a/legacy/candle/src/inference.rs b/legacy/candle/src/inference.rs index 0e34efd..397d4ec 100644 --- a/legacy/candle/src/inference.rs +++ b/legacy/candle/src/inference.rs @@ -1,8 +1,8 @@ //! Cydonia inference interface use anyhow::Result; -use candle_core::{quantized::gguf_file::Content, Device, Tensor}; +use candle_core::{Device, Tensor, quantized::gguf_file::Content}; use candle_transformers::models::quantized_llama; -use model::{chat, Message}; +use model::{Message, chat}; use std::fs::File; /// The inference interface for language models diff --git a/legacy/model/Cargo.toml b/legacy/model/Cargo.toml index fe22c22..c27941f 100644 --- a/legacy/model/Cargo.toml +++ b/legacy/model/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "cydonia-model" -version = "0.0.0" -edition = "2021" +description = "Cydonia model utils" +documentation = "https://docs.rs/cydonia-model" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true [dependencies] anyhow.workspace = true diff --git a/llm/deepseek/Cargo.toml b/llm/deepseek/Cargo.toml index 613940f..de005cc 100644 --- a/llm/deepseek/Cargo.toml +++ b/llm/deepseek/Cargo.toml @@ -1,15 +1,16 @@ [package] -name = "ullm-deepseek" +name = "cydonia-deepseek" +description = "Cydonia DeepSeek provider implementation" +documentation = "https://docs.rs/cydonia-deepseek" version.workspace = true edition.workspace = true authors.workspace = true license.workspace = true repository.workspace = true -documentation.workspace = true keywords.workspace = true [dependencies] -ucore.workspace = true +ccore.workspace = true # crates-io dependencies anyhow.workspace = true diff --git a/llm/deepseek/README.md b/llm/deepseek/README.md new file mode 100644 index 0000000..87d932b --- /dev/null +++ b/llm/deepseek/README.md @@ -0,0 +1,24 @@ +# cydonia-deepseek + +DeepSeek LLM provider for cydonia. + +## Overview + +Implements the `LLM` trait for DeepSeek API, supporting: + +- `deepseek-chat` - Standard chat model +- `deepseek-reasoner` - Reasoning model with thinking mode +- Streaming responses +- Tool calling with thinking mode + +## Usage + +```rust +use cydonia::{DeepSeek, LLM}; + +let provider = DeepSeek::new(client, "your-api-key")?; +``` + +## License + +MIT diff --git a/llm/deepseek/src/lib.rs b/llm/deepseek/src/lib.rs index c4c9461..23c3315 100644 --- a/llm/deepseek/src/lib.rs +++ b/llm/deepseek/src/lib.rs @@ -1,7 +1,7 @@ //! The LLM provider +use ccore::{Client, reqwest::header::HeaderMap}; pub use request::Request; -use ucore::{Client, reqwest::header::HeaderMap}; mod llm; mod request; diff --git a/llm/deepseek/src/llm.rs b/llm/deepseek/src/llm.rs index 93c8a75..472cceb 100644 --- a/llm/deepseek/src/llm.rs +++ b/llm/deepseek/src/llm.rs @@ -3,15 +3,15 @@ use crate::{DeepSeek, Request}; use anyhow::Result; use async_stream::try_stream; -use futures_core::Stream; -use futures_util::StreamExt; -use ucore::{ - ChatMessage, Client, LLM, Response, StreamChunk, +use ccore::{ + Client, LLM, Message, Response, StreamChunk, reqwest::{ Method, header::{self, HeaderMap}, }, }; +use futures_core::Stream; +use futures_util::StreamExt; const ENDPOINT: &str = "https://api.deepseek.com/chat/completions"; @@ -29,17 +29,20 @@ impl LLM for DeepSeek { } /// Send a message to the LLM - async fn send(&mut self, req: &Request, messages: &[ChatMessage]) -> Result { + async fn send(&mut self, req: &Request, messages: &[Message]) -> Result { + let body = req.messages(messages); + tracing::debug!("request: {}", serde_json::to_string(&body)?); let text = self .client .request(Method::POST, ENDPOINT) .headers(self.headers.clone()) - .json(&req.messages(messages)) + .json(&body) .send() .await? .text() .await?; + tracing::debug!("response: {text}"); serde_json::from_str(&text).map_err(Into::into) // self.client // .request(Method::POST, ENDPOINT) @@ -56,21 +59,30 @@ impl LLM for DeepSeek { fn stream( &mut self, req: Request, - messages: &[ChatMessage], + messages: &[Message], usage: bool, ) -> impl Stream> { + let body = req.messages(messages).stream(usage); + tracing::debug!( + "request: {}", + serde_json::to_string(&body).unwrap_or_default() + ); let request = self .client .request(Method::POST, ENDPOINT) .headers(self.headers.clone()) - .json(&req.messages(messages).stream(usage)); + .json(&body); try_stream! { let mut stream = request.send().await?.bytes_stream(); while let Some(chunk) = stream.next().await { let text = String::from_utf8_lossy(&chunk?).into_owned(); for data in text.split("data: ").skip(1).filter(|s| !s.starts_with("[DONE]")) { - yield serde_json::from_str(data.trim())?; + tracing::debug!("response: {}", data.trim()); + match serde_json::from_str::(data.trim()) { + Ok(chunk) => yield chunk, + Err(e) => tracing::warn!("failed to parse chunk: {e}, data: {}", data.trim()), + } } } } diff --git a/llm/deepseek/src/request.rs b/llm/deepseek/src/request.rs index 2fc10d4..cbf49c3 100644 --- a/llm/deepseek/src/request.rs +++ b/llm/deepseek/src/request.rs @@ -1,14 +1,14 @@ //! The request body for the DeepSeek API +use ccore::{Config, General, Message, Tool, ToolChoice}; use serde::Serialize; use serde_json::{Value, json}; -use ucore::{ChatMessage, Config, General, Tool, ToolChoice}; /// The request body for the DeepSeek API #[derive(Debug, Clone, Serialize)] pub struct Request { /// The messages to send to the API - pub messages: Vec, + pub messages: Vec, /// The model we are using pub model: String, @@ -73,7 +73,7 @@ pub struct Request { impl Request { /// Construct the messages for the request - pub fn messages(&self, messages: &[ChatMessage]) -> Self { + pub fn messages(&self, messages: &[Message]) -> Self { Self { messages: messages.to_vec(), ..self.clone() @@ -105,7 +105,13 @@ impl From for Request { stop: None, stream: None, stream_options: None, - thinking: None, + thinking: if config.think { + Some(json!({ + "type": "enabled" + })) + } else { + None + }, temperature: None, tool_choice: None, tools: None, @@ -117,6 +123,15 @@ impl From for Request { impl Config for Request { fn with_tools(self, tools: Vec) -> Self { + let tools = tools + .into_iter() + .map(|tool| { + json!({ + "type": "function", + "function": json!(tool), + }) + }) + .collect::>(); Self { tools: Some(json!(tools)), ..self.clone()