Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/current.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ Record image-affecting changes to `manager/`, `worker/`, `copaw/`, `openclaw-bas
- **Legacy Team channel policy**: Legacy Team reconciliation now writes final Matrix channel allow-lists to member runtime config and re-adds the Team Leader to the Manager allow-list so controller integration tests observe durable policy state.
- **Sandbox worker-deps hardening**: Sandbox-backed Workers now prepare controller-owned worker-deps env/token/data material before claim creation, recycle stale SandboxClaims and bound Sandboxes on runtime-affecting changes, and use bounded ServiceAccount token projection for built-in SandboxClaim mounts.
- **CLI AgentTeams auth env**: The `hiclaw` CLI now discovers `AGENTTEAMS_CONTROLLER_URL`, `AGENTTEAMS_AUTH_TOKEN`, `AGENTTEAMS_AUTH_TOKEN_FILE`, and `AGENTTEAMS_CLUSTER_ID` while preserving legacy `HICLAW_*` fallbacks, so Manager and Worker containers can use the terminal env names for controller calls.
- **OpenClaw Worker file-sync token safety**: The Worker file-sync skill now keeps `openclaw.json` out of bulk mirrors and merges a separately fetched remote copy, preventing stale Matrix tokens from being briefly restored during sync.
- **CoPaw worker runtime environment**: CoPaw workers now prefer AgentTeams storage/runtime environment variables while preserving legacy HiClaw fallbacks, and Qwen-style model health preflights disable thinking for lightweight readiness checks.
- **CoPaw Worker heartbeat**: CoPaw worker templates now seed heartbeat at a 10-minute interval so Team Leader agents created from the worker template can run heartbeat turns without requiring an explicit Team CR heartbeat spec.
- **Helm CRDs**: Removed unsupported `propertyNames` schema fields from Worker and Team CRDs so Kubernetes API servers accept the chart CRDs.
Expand Down
19 changes: 10 additions & 9 deletions manager/agent/worker-agent/skills/file-sync/scripts/hiclaw-sync.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,25 @@ WORKSPACE="${AGENTTEAMS_ROOT}/agents/${WORKER_NAME}"

ensure_mc_credentials 2>/dev/null || true

# Save local openclaw.json before mirror overwrites it
# Pull remote openclaw.json separately so bulk mirror never exposes the
# stale MinIO token to OpenClaw's live config watcher.
LOCAL_OPENCLAW="${WORKSPACE}/openclaw.json"
SAVED_LOCAL="/tmp/openclaw-local-sync.json"
if [ -f "${LOCAL_OPENCLAW}" ]; then
cp "${LOCAL_OPENCLAW}" "${SAVED_LOCAL}"
fi
REMOTE_OPENCLAW="/tmp/openclaw-remote-sync.json"
rm -f "${REMOTE_OPENCLAW}"
mc cp "${AGENTTEAMS_STORAGE_PREFIX}/agents/${WORKER_NAME}/openclaw.json" "${REMOTE_OPENCLAW}" 2>/dev/null || true

mc mirror "${AGENTTEAMS_STORAGE_PREFIX}/agents/${WORKER_NAME}/" "${WORKSPACE}/" --overwrite \
--exclude "openclaw.json" \
--exclude ".openclaw/matrix/**" --exclude ".openclaw/canvas/**" 2>&1
mc mirror "${AGENTTEAMS_STORAGE_PREFIX}/shared/" "${AGENTTEAMS_ROOT}/shared/" --overwrite 2>/dev/null || true

# Update pull marker so the local→remote sync loop doesn't push back freshly-pulled files
touch "${WORKSPACE}/.last-pull"

# Merge openclaw.json: local-first (pre-mirror copy) with MinIO overlay (arg1=remote, arg2=local, arg3=out)
if [ -f "${SAVED_LOCAL}" ] && [ -f "${LOCAL_OPENCLAW}" ]; then
merge_openclaw_config "${LOCAL_OPENCLAW}" "${SAVED_LOCAL}" "${LOCAL_OPENCLAW}"
rm -f "${SAVED_LOCAL}"
# Merge openclaw.json: local-first with MinIO overlay (arg1=remote, arg2=local, arg3=out)
if [ -f "${REMOTE_OPENCLAW}" ]; then
merge_openclaw_config "${REMOTE_OPENCLAW}" "${LOCAL_OPENCLAW}" "${LOCAL_OPENCLAW}"
rm -f "${REMOTE_OPENCLAW}"
fi

# Restore +x on scripts (MinIO does not preserve Unix permission bits)
Expand Down
58 changes: 18 additions & 40 deletions tests/test-06-multi-worker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,53 +35,31 @@ wait_for_manager_agent_ready 300 "${DM_ROOM}" "${ADMIN_TOKEN}" || {
exit 1
}

# test-05 can leave CoPaw Manager finishing heartbeat / pending-worker cleanup
# replies in the admin DM. Let that prior turn go quiet before measuring Bob's
# create-worker ack/provisioning SLA; the post-request waits below stay strict.
if ! matrix_wait_for_sender_quiet "${ADMIN_TOKEN}" "${DM_ROOM}" "@manager" 20 180; then
log_fail "Manager DM did not become quiet before Bob create request"
test_teardown "06-multi-worker"
test_summary
exit 1
fi

# Alice is running from previous tests; bob will be created below (offset=0 is correct for new workers)
wait_for_worker_container "alice" 60
METRICS_BASELINE=$(snapshot_baseline "alice" "bob")
TEST_WORKER_RUNTIME="${HICLAW_DEFAULT_WORKER_RUNTIME:-openclaw}"
# worker-management/SKILL.md tells Manager to ask admin for FOUR inputs
# (name / runtime / SOUL / skills) before running `hiclaw create worker`
# and not to invent defaults. A vague prompt that only names the worker is
# therefore a coin flip — sometimes Manager replies with a confirmation
# request, never calls the CLI, and the consumer/SOUL.md polls below
# silently time out. Spell out all four inputs and tell Manager to skip
# confirmation so this test exercises actual Worker creation.
#
# The runtime is explicit because the CI matrix runtime is the source of truth;
# rendered Manager workspace text may contain fallback defaults.
matrix_send_message "${ADMIN_TOKEN}" "${DM_ROOM}" \
"Please create a new Worker now using these exact values — do not ask me to confirm any of them:
- name: bob
- runtime: ${TEST_WORKER_RUNTIME} (use this exact runtime; do not reinterpret it as the install default)
- SOUL/role: Backend developer specializing in REST APIs, server-side logic, and data persistence
- skills: github-operations (file-sync / task-progress / project-participation are auto-included, no need to ask)

Proceed immediately and tell me when he is created."

log_info "Waiting for Manager to create Worker Bob..."
REPLY=$(matrix_wait_for_reply_matching "${ADMIN_TOKEN}" "${DM_ROOM}" "@manager" \
"bob.*(accepted|created|creating|pending|running|ready)" 300 \
"${ADMIN_TOKEN}" "${DM_ROOM}" "Please check if the request to create worker bob has been processed.")

assert_not_empty "${REPLY}" "Manager replied to create bob request"
assert_contains_i "${REPLY}" "bob" "Reply mentions worker name 'bob'"
# This test verifies multi-worker collaboration after Alice and Bob exist. Bob
# creation must be deterministic setup: asking the Manager to create him via DM
# is flaky under CI because the Manager may still be processing heartbeat/task
# follow-ups from earlier shard-A tests, or may hit a tool-guard prompt before
# calling `hiclaw create worker`. Use the controller CLI directly here so the
# collaboration assertions below measure the multi-worker flow, not LLM timing.
BOB_SOUL="Backend developer specializing in REST APIs, server-side logic, and data persistence"
log_info "Creating Worker Bob via hiclaw CLI (runtime: ${TEST_WORKER_RUNTIME})..."
CREATE_OUTPUT=$(exec_in_agent hiclaw apply worker --name bob \
--runtime "${TEST_WORKER_RUNTIME}" \
--soul "${BOB_SOUL}" \
--skills github-operations 2>&1)
if echo "${CREATE_OUTPUT}" | grep -qiE "worker/bob (created|configured)"; then
log_pass "hiclaw apply worker bob accepted"
else
log_fail "hiclaw apply worker bob failed: ${CREATE_OUTPUT}"
fi

# Verify Bob's infrastructure. Worker creation is asynchronous, so wait on
# persisted provisioning state and gateway side effects instead of sleeping.
BOB_PROVISION_TIMEOUT=60
if echo "${REPLY}" | grep -qiE "bob.*(accepted|creating|pending)" 2>/dev/null; then
BOB_PROVISION_TIMEOUT=180
fi
BOB_PROVISION_TIMEOUT=180
if wait_worker_provisioned "bob" "${BOB_PROVISION_TIMEOUT}"; then
log_pass "Worker Bob provisioned (roomID + matrixUserID populated)"
else
Expand Down
7 changes: 7 additions & 0 deletions tests/test-17-worker-config-verify.sh
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,13 @@ else
log_fail "file-sync SKILL.md does not use 'coordinator'"
fi

FILESYNC_SCRIPT=$(exec_in_manager mc cat "${STORAGE_PREFIX}/agents/${TEST_WORKER}/skills/file-sync/scripts/hiclaw-sync.sh" 2>/dev/null || echo "")
if echo "${FILESYNC_SCRIPT}" | grep -q -- '--exclude "openclaw.json"'; then
log_pass "file-sync script preserves local openclaw.json during mirror"
else
log_fail "file-sync script does not exclude openclaw.json from bulk mirror"
fi

# ============================================================
# Section 4: Verify other MinIO artifacts
# ============================================================
Expand Down
4 changes: 4 additions & 0 deletions tests/test-18-team-config-verify.sh
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,10 @@ assert_eq "worker" "${W2_ROLE}" "Worker 2 has role=worker"
# ============================================================
log_section "Verify Leader AGENTS.md"

# TeamReconciler injects the leader context after member provisioning and room
# IDs are available. Avoid a single early MinIO read racing that async write.
wait_agent_file_contains "${TEST_LEADER}" "AGENTS.md" "hiclaw-team-context-start" 120 || true

LEADER_AGENTS=$(exec_in_manager mc cat "${STORAGE_PREFIX}/agents/${TEST_LEADER}/AGENTS.md" 2>/dev/null || echo "")
assert_not_empty "${LEADER_AGENTS}" "Leader AGENTS.md exists in MinIO"

Expand Down
7 changes: 7 additions & 0 deletions tests/test-21-team-project-dag.sh
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,7 @@ MAX_DM_ONLY_POLLS="${MAX_DM_ONLY_POLLS:-1}"

LEADER_RESPONDED=false
TEAM_COORDINATED=false
LEADER_COORDINATION_NUDGE_SENT=false
RUNTIME_ERROR=false
DM_ONLY_POLLS=0
for i in $(seq 1 "${MAX_COORDINATION_POLLS}"); do
Expand Down Expand Up @@ -394,6 +395,12 @@ for i in $(seq 1 "${MAX_COORDINATION_POLLS}"); do
if echo "${DM_MSGS}" | grep -qi "Error:\\|No active model configured"; then
RUNTIME_ERROR=true
fi
if [ "${LEADER_COORDINATION_NUDGE_SENT}" != "true" ] && [ "${TEAM_COORDINATED}" != "true" ]; then
log_info "Leader is active in DM but not coordinating in Team Room; sending one corrective Team Room nudge"
matrix_send_message "${ADMIN_LOGIN_TOKEN}" "${TEAM_ROOM}" \
"@${TEST_LEADER}:${TEST_MATRIX_DOMAIN} You are the Team Leader, not the implementer. Stop doing the API work yourself. In this Team Room, create/delegate tasks and @mention ${TEST_W1} for API design/implementation and ${TEST_W2} for QA/test cases. Workers only process task assignments addressed to them in this Team Room."
LEADER_COORDINATION_NUDGE_SENT=true
fi
fi

if [ "${LEADER_RESPONDED}" = "true" ] && [ "${TEAM_COORDINATED}" != "true" ]; then
Expand Down
Loading