fix(agents): handle UTF-16 encoded agent config files

Fixes bounty issue #1469

Agent configuration files encoded as UTF-16 (common on Windows) would
fail with an unhelpful 'stream did not contain valid UTF-8' error
when using std::fs::read_to_string().

This change uses the existing text_encoding utilities to detect and
decode various file encodings including UTF-16LE and UTF-16BE, providing
clearer error messages when encoding issues occur.

Author: factorydroid

cortex-cli/src/agent_cmd.rs

@@ -7,11 +7,32 @@
 
 use anyhow::{Context, Result, bail};
 use clap::Parser;
+use cortex_engine::text_encoding::{Encoding, decode};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::io::{self, BufRead, Write};
 use std::path::{Path, PathBuf};
 
+/// Read a file's contents, automatically detecting and handling various text encodings.
+///
+/// This function reads raw bytes from the file, detects the encoding (including UTF-16),
+/// and decodes it to a UTF-8 string. This is more robust than `std::fs::read_to_string`
+/// which only handles UTF-8.
+fn read_file_with_encoding(path: &Path) -> Result<String> {
+    let bytes =
+        std::fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?;
+    let encoding = Encoding::detect(&bytes);
+    decode(&bytes, encoding)
+        .map(|cow| cow.into_owned())
+        .with_context(|| {
+            format!(
+                "Failed to decode file '{}' with encoding {}",
+                path.display(),
+                encoding.name()
+            )
+        })
+}
+
 /// Agent management CLI.
 #[derive(Debug, Parser)]
 pub struct AgentCli {
@@ -613,8 +634,7 @@ fn load_agents_from_dir(dir: &Path, source: AgentSource) -> Result<Vec<AgentInfo
 
 /// Load an agent from a markdown file with YAML frontmatter.
 fn load_agent_from_md(path: &Path, source: AgentSource) -> Result<AgentInfo> {
-    let content = std::fs::read_to_string(path)
-        .with_context(|| format!("Failed to read {}", path.display()))?;
+    let content = read_file_with_encoding(path)?;
 
     let (frontmatter, body) = parse_frontmatter(&content)?;
 
@@ -643,8 +663,7 @@ fn load_agent_from_md(path: &Path, source: AgentSource) -> Result<AgentInfo> {
 
 /// Load an agent from a JSON file.
 fn load_agent_from_json(path: &Path, source: AgentSource) -> Result<AgentInfo> {
-    let content = std::fs::read_to_string(path)
-        .with_context(|| format!("Failed to read {}", path.display()))?;
+    let content = read_file_with_encoding(path)?;
 
     let frontmatter: AgentFrontmatter = serde_json::from_str(&content)
         .with_context(|| format!("Failed to parse {}", path.display()))?;
@@ -653,7 +672,7 @@ fn load_agent_from_json(path: &Path, source: AgentSource) -> Result<AgentInfo> {
     let prompt = if let Some(parent) = path.parent() {
         let prompt_file = parent.join("prompt.md");
         if prompt_file.exists() {
-            Some(std::fs::read_to_string(&prompt_file)?)
+            Some(read_file_with_encoding(&prompt_file)?)
         } else {
             None
         }

cortex-engine/src/agents.rs

@@ -25,6 +25,27 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::RwLock;
 
 use crate::error::{CortexError, Result};
+use crate::text_encoding::{Encoding, decode};
+
+/// Read a file's contents, automatically detecting and handling various text encodings.
+///
+/// This function reads raw bytes from the file, detects the encoding (including UTF-16),
+/// and decodes it to a UTF-8 string. This is more robust than `std::fs::read_to_string`
+/// which only handles UTF-8.
+fn read_file_with_encoding(path: &Path) -> Result<String> {
+    let bytes = std::fs::read(path)?;
+    let encoding = Encoding::detect(&bytes);
+    decode(&bytes, encoding)
+        .map(|cow| cow.into_owned())
+        .map_err(|e| {
+            CortexError::InvalidInput(format!(
+                "Failed to decode file '{}' with encoding {}: {}",
+                path.display(),
+                encoding.name(),
+                e
+            ))
+        })
+}
 
 // ============================================================================
 // OS-Specific Agents Directory
@@ -461,15 +482,15 @@ impl AgentRegistry {
         agent_md: &Path,
         source: AgentSource,
     ) -> Result<Agent> {
-        let content = std::fs::read_to_string(agent_md)?;
+        let content = read_file_with_encoding(agent_md)?;
         let (metadata, prompt) = parse_agent_md(&content)?;
 
         metadata.validate()?;
 
         // Resolve system prompt
         let system_prompt = if let Some(ref prompt_file) = metadata.prompt_file {
             let prompt_path = agent_dir.join(prompt_file);
-            std::fs::read_to_string(&prompt_path)?
+            read_file_with_encoding(&prompt_path)?
         } else if let Some(ref prompt) = metadata.system_prompt {
             prompt.clone()
         } else {
@@ -491,15 +512,15 @@ impl AgentRegistry {
         agent_json: &Path,
         source: AgentSource,
     ) -> Result<Agent> {
-        let content = std::fs::read_to_string(agent_json)?;
+        let content = read_file_with_encoding(agent_json)?;
         let metadata: AgentMetadata = serde_json::from_str(&content)?;
 
         metadata.validate()?;
 
         // Resolve system prompt
         let system_prompt = if let Some(ref prompt_file) = metadata.prompt_file {
             let prompt_path = agent_dir.join(prompt_file);
-            std::fs::read_to_string(&prompt_path)?
+            read_file_with_encoding(&prompt_path)?
         } else {
             metadata.system_prompt.clone().unwrap_or_default()
         };