fix: resolve operator, agent networking, auth, alerting, and tracing issues

cmd/api/enterprise.go

@@ -369,6 +369,16 @@ var defaultRoles = map[string]Role{
 			{Resource: "agents", Verbs: []string{"read"}},
 			{Resource: "health", Verbs: []string{"read"}},
 			{Resource: "routes", Verbs: []string{"read"}},
+			{Resource: "skills", Verbs: []string{"read"}},
+			{Resource: "tools", Verbs: []string{"read"}},
+			{Resource: "catalog", Verbs: []string{"read"}},
+			{Resource: "models", Verbs: []string{"read"}},
+			{Resource: "connectors", Verbs: []string{"read"}},
+			{Resource: "rules", Verbs: []string{"read"}},
+			{Resource: "memory", Verbs: []string{"read"}},
+			{Resource: "stats", Verbs: []string{"read"}},
+			{Resource: "eval", Verbs: []string{"read"}},
+			{Resource: "costs", Verbs: []string{"read"}},
 		},
 	},
 }

cmd/api/main.go

@@ -3,6 +3,8 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"log"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -1538,6 +1540,18 @@ func main() {
 
 	enterprise.RegisterRoutes(httpSrv)
 
+	httpSrv.HandleFunc("/api/v1/webhooks/alerts/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		severity := strings.TrimPrefix(r.URL.Path, "/api/v1/webhooks/alerts/")
+		body, _ := io.ReadAll(r.Body)
+		log.Printf("alert webhook received: severity=%s payload=%s", severity, string(body))
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, `{"status":"accepted","severity":%q}`, severity)
+	})
+
 	httpSrv.HandleFunc("/api/v1/health", enterprise.AuthMiddleware(cors(healthHandler)))
 	httpSrv.HandleFunc("/api/v1/routes", enterprise.AuthMiddleware(cors(routesHandler)))
 	httpSrv.HandleFunc("/api/v1/chat", enterprise.AuthMiddleware(chatHandler))

cmd/mesh/handlers.go

@@ -171,6 +171,17 @@ func (s *Server) provisionAgentNamespace(agent *MeshAgent) {
 		s.provisionDeepAgentPod(ns, agent)
 	}
 
+	baseDomain := os.Getenv("ARCANA_BASE_DOMAIN")
+	if baseDomain == "" {
+		baseDomain = "arcana.localhost.me"
+	}
+	agentHost := agent.Name + "." + baseDomain
+	if err := s.k8s.CreateIngress(ns, "agent-ingress", agentHost, "deep-agent", 5002); err != nil {
+		log.Printf("failed to create ingress in %s: %v", ns, err)
+	} else {
+		log.Printf("ingress created: %s → deep-agent:5002 in %s", agentHost, ns)
+	}
+
 	log.Printf("agent %s fully provisioned in namespace %s", agent.Name, ns)
 }
 

cmd/mesh/k8s.go

@@ -164,6 +164,18 @@ func (k *K8sClient) CreateNetworkPolicy(namespace, name string) error {
 						}},
 					},
 				},
+				{
+					"from": []map[string]interface{}{
+						{"namespaceSelector": map[string]interface{}{
+							"matchLabels": map[string]string{
+								"app.kubernetes.io/name": "ingress-nginx",
+							},
+						}},
+					},
+					"ports": []map[string]interface{}{
+						{"protocol": "TCP", "port": 5002},
+					},
+				},
 			},
 			"egress": []map[string]interface{}{
 				{
@@ -184,6 +196,11 @@ func (k *K8sClient) CreateNetworkPolicy(namespace, name string) error {
 						{"protocol": "UDP", "port": 53},
 					},
 				},
+				{
+					"ports": []map[string]interface{}{
+						{"protocol": "TCP", "port": 443},
+					},
+				},
 			},
 		},
 	}
@@ -314,6 +331,59 @@ func (k *K8sClient) CreateService(namespace, name string, port int, targetPort i
 	return nil
 }
 
+func (k *K8sClient) CreateIngress(namespace, name, host, serviceName string, servicePort int) error {
+	ing := map[string]interface{}{
+		"apiVersion": "networking.k8s.io/v1",
+		"kind":       "Ingress",
+		"metadata": map[string]interface{}{
+			"name":      name,
+			"namespace": namespace,
+			"annotations": map[string]string{
+				"nginx.ingress.kubernetes.io/proxy-read-timeout": "3600",
+				"nginx.ingress.kubernetes.io/proxy-send-timeout": "3600",
+				"nginx.ingress.kubernetes.io/proxy-buffering":    "off",
+				"nginx.ingress.kubernetes.io/ssl-redirect":       "false",
+			},
+		},
+		"spec": map[string]interface{}{
+			"ingressClassName": "nginx",
+			"rules": []map[string]interface{}{
+				{
+					"host": host,
+					"http": map[string]interface{}{
+						"paths": []map[string]interface{}{
+							{
+								"path":     "/",
+								"pathType": "Prefix",
+								"backend": map[string]interface{}{
+									"service": map[string]interface{}{
+										"name": serviceName,
+										"port": map[string]interface{}{
+											"number": servicePort,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	_, code, err := k.do("POST",
+		fmt.Sprintf("/apis/networking.k8s.io/v1/namespaces/%s/ingresses", namespace), ing)
+	if err != nil {
+		return err
+	}
+	if code == 409 {
+		return nil
+	}
+	if code >= 300 {
+		return fmt.Errorf("create ingress: HTTP %d", code)
+	}
+	return nil
+}
+
 func (k *K8sClient) GetDeploymentStatus(namespace, name string) (map[string]interface{}, error) {
 	body, code, err := k.do("GET",
 		fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/%s", namespace, name), nil)

cmd/operator/main.go

@@ -17,7 +17,7 @@ import (
 )
 
 var agentGVR = schema.GroupVersionResource{
-	Group:    "arcana.ai",
+	Group:    "arcana.io",
 	Version:  "v1alpha1",
 	Resource: "arcanaagents",
 }
@@ -81,7 +81,7 @@ func reconcile(client dynamic.Interface) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	agents, err := client.Resource(agentGVR).Namespace("arcana").List(ctx, metav1.ListOptions{})
+	agents, err := client.Resource(agentGVR).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		log.Printf("operator: list agents failed: %v", err)
 		status.Errors = append(status.Errors, fmt.Sprintf("%s: %v", time.Now().Format(time.RFC3339), err))

deploy/backing/ingress.yaml

@@ -29,13 +29,6 @@ spec:
                 name: arcana-api
                 port:
                   number: 8080
-          - path: /agents/
-            pathType: Prefix
-            backend:
-              service:
-                name: arcana-api
-                port:
-                  number: 8080
           - path: /events
             pathType: Prefix
             backend:

deploy/helm/arcana-agui/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8084"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-annotate/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8102"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-api/values.yaml

@@ -46,7 +46,7 @@ env:
   REGISTRY_HOST: "arcana-registry.arcana.svc.cluster.local"
   FINOPS_HOST: "arcana-finops.arcana.svc.cluster.local"
   GITOPS_HOST: "arcana-gitops.arcana.svc.cluster.local"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-audit/values.yaml

@@ -21,7 +21,7 @@ env:
   PORT: "8100"
   POSTGRES_HOST: "postgres.arcana.svc.cluster.local"
   POSTGRES_PORT: "5432"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-blueprint/values.yaml

@@ -21,7 +21,7 @@ env:
   PORT: "8088"
   POSTGRES_HOST: "postgres.arcana.svc.cluster.local"
   POSTGRES_PORT: "5432"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-codex-ingestor/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8092"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-codex-router/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8090"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-codex-scorer/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8093"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-codex-searcher/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8091"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-connectors/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8094"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-engine/values.yaml

@@ -24,7 +24,7 @@ env:
   REDIS_HOST: "redis.arcana.svc.cluster.local"
   NATS_HOST: "nats.arcana.svc.cluster.local"
   TEMPORAL_HOST: "temporal.arcana.svc.cluster.local"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-finops/values.yaml

@@ -21,7 +21,7 @@ env:
   PORT: "8105"
   POSTGRES_HOST: "postgres.arcana.svc.cluster.local"
   POSTGRES_PORT: "5432"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-forge/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8098"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-gitops/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8106"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-graph/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8095"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-memory/values.yaml

@@ -21,7 +21,7 @@ env:
   PORT: "8087"
   POSTGRES_HOST: "postgres.arcana.svc.cluster.local"
   POSTGRES_PORT: "5432"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-mesh/values.yaml

@@ -22,7 +22,7 @@ env:
   NATS_URL: "nats://nats.arcana.svc.cluster.local:4222"
   POSTGRES_HOST: "postgres.arcana.svc.cluster.local"
   POSTGRES_PORT: "5432"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
   TEMPLATE_AGENT_IMAGE: "ghcr.io/redhat-data-and-ai/template-agent:deep-agent"
   TEMPLATE_AGENT_PULL_POLICY: "Always"
 

deploy/helm/arcana-models/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8099"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-operator/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8082"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-oracle/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8089"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-probe/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8101"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 probes:
   liveness: "/healthz"

deploy/helm/arcana-registry/values.yaml

@@ -21,7 +21,7 @@ env:
   PORT: "8104"
   POSTGRES_HOST: "postgres.arcana.svc.cluster.local"
   POSTGRES_PORT: "5432"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
 
 envFromSecrets:
   - arcana-platform-secret

deploy/helm/arcana-sandbox/values.yaml

@@ -19,7 +19,7 @@ service:
 
 env:
   PORT: "8097"
-  OTEL_EXPORTER_OTLP_ENDPOINT: "jaeger.arcana.svc.cluster.local:4318"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger.arcana.svc.cluster.local:4318"
   SANDBOX_RUNTIME_CLASS: "arcana-sandbox-dev"
   SANDBOX_NAMESPACE: "arcana"