consul connect envoy using default token instead of specified token

[mattclement]

Overview of the Issue

Removing service:write from default token is causing errors on consul connect envoy, although a separate token with service:write is specified for the command.

---

Reproduction Steps

1. Start with a default token with write permissions for everything.
2. Start consul connect envoy -sidecar-for=service or consul connect envoy -gateway=ingress with an ACL token with the following policy:

agent_prefix "" {
  policy = "read"
}

service_prefix "" {
  policy = "write"
}

mesh = "write"

3. Apply this restrictive policy to the default token:

node_prefix "" {
  policy = "read"
}

operator = "read"

query_prefix "" {
  policy = "read"
}

service_prefix "" {
  policy = "read"
}

4. Observe errors indicating the default token is being used instead of the token specified for the envoy proxy.

Consul info for both Client and Server

Client info

We run a custom fork of consul that has additional logging and adjusts some timeouts, no behavioral changes related to ACLs:

agent:
        check_monitors = 2
        check_ttls = 0
        checks = 8
        services = 7
build:
        prerelease = company
        revision =
        version = 1.20.1
        version_metadata =
consul:
        acl = enabled
        known_servers = 3
        server = false
runtime:
        arch = amd64
        cpu_count = 2
        goroutines = 211
        max_procs = 2
        os = linux
        version = go1.22.5
serf_lan:
        coordinate_resets = 0
        encrypted = false
        event_queue = 0
        event_time = 472
        failed = 0
        health_score = 0
        intent_queue = 0
        left = 0
        member_time = 588665
        members = 166
        query_queue = 0
        query_time = 1

{
    "acl": {
        "default_policy": "deny",
        "down_policy": "async-cache",
        "enable_token_persistence": true,
        "enable_token_replication": true,
        "enabled": true,
        "tokens": {
            "agent": "token",
            "config_file_service_registration": "token",
            "default": "token"
        }
    },
    "addresses": {
        "dns": "127.0.0.1",
        "grpc": "127.0.0.1",
        "http": "127.0.0.1"
    },
    "advertise_addr": "{{ GetInterfaceIP \"ens4\" }}",
    "auto_reload_config": true,
    "config_entries": {

    },
    "connect": {
        "ca_config": {
            "csr_max_concurrent": 0,
            "csr_max_per_second": 50,
            "leaf_cert_ttl": "72h"
        },
        "enabled": true
    },
    "data_dir": "/opt/consul",
    "datacenter": "datacenter-2",
    "default_intention_policy": "allow",
    "disable_remote_exec": true,
    "disable_update_check": true,
    "dns_config": {
        "allow_stale": true,
        "max_stale": "87600h",
        "service_ttl": {
            "*": "2s",
            "service-name": "5s",
            "service-name": "5s",
            "service-name": "5s",
            "service-name": "10s",
            "service-name": "2s",
            "service-name": "5s"
        }
    },
    "enable_central_service_config": true,
    "enable_script_checks": true,
    "enable_syslog": true,
    "log_level": "INFO",
    "node_meta": {
        "cluster": "cluster",
        "machine_type": "machine-type",
        "zone": "zone"
    },
    "node_name": "node-name",
    "ports": {
        "dns": 8600,
        "grpc": 8502,
        "http": 8500,
        "serf_lan": 8301,
        "serf_wan": 8302
    },
    "primary_datacenter": "datacenter",
    "retry_join": [
        "provider=gce tag_value=value zone_pattern=us-central1-.*"
    ],
    "retry_join_wan": [

    ],
    "tls": {
        "defaults": {
            "ca_file": "/etc/consul/ssl/ca.pem",
            "tls_min_version": "TLSv1_2",
            "verify_incoming": false,
            "verify_outgoing": false
        }
    },
    "use_streaming_backend": false
}

Server info

agent:
        check_monitors = 2
        check_ttls = 0
        checks = 6
        services = 5
build:
        prerelease = company
        revision =
        version = 1.20.1
        version_metadata =
consul:
        acl = enabled
        bootstrap = false
        known_datacenters = 5
        leader = true
        leader_addr = leader_ip:8300
        server = true
raft:
        applied_index = 1363425798
        commit_index = 1363425798
        fsm_pending = 0
        last_contact = 0
        last_log_index = 1363425798
        last_log_term = 872
        last_snapshot_index = 1363414104
        last_snapshot_term = 872
        latest_configuration = [{Suffrage:Voter ID:id1 Address:leader_ip:8300} {Suffrage:Voter ID:id2 Address:second_ip:8300} {Suffrage:Voter ID:id3 Address:third_ip:8300}]
        latest_configuration_index = 0
        num_peers = 2
        protocol_version = 3
        protocol_version_max = 3
        protocol_version_min = 0
        snapshot_version_max = 1
        snapshot_version_min = 0
        state = Leader
        term = 872
runtime:
        arch = amd64
        cpu_count = 2
        goroutines = 5059
        max_procs = 2
        os = linux
        version = go1.22.5
serf_lan:
        coordinate_resets = 0
        encrypted = false
        event_queue = 0
        event_time = 472
        failed = 0
        health_score = 0
        intent_queue = 0
        left = 0
        member_time = 588665
        members = 166
        query_queue = 0
        query_time = 1
serf_wan:
        coordinate_resets = 0
        encrypted = false
        event_queue = 0
        event_time = 1
        failed = 0
        health_score = 0
        intent_queue = 0
        left = 0
        member_time = 34563
        members = 15
        query_queue = 0
        query_time = 1

{
    "acl": {
        "default_policy": "deny",
        "down_policy": "async-cache",
        "enable_token_persistence": true,
        "enable_token_replication": true,
        "enabled": true,
        "tokens": {
            "agent": "token",
            "config_file_service_registration": "token",
            "default": "token",
            "initial_management": "token",
            "replication": "token"
        }
    },
    "addresses": {
        "dns": "0.0.0.0",
        "grpc": "0.0.0.0",
        "grpc_tls": "0.0.0.0",
        "http": "0.0.0.0",
        "https": "0.0.0.0"
    },
    "advertise_addr": "{{ GetInterfaceIP \"ens4\" }}",
    "auto_reload_config": true,
    "bootstrap_expect": 3,
    "config_entries": {
        "bootstrap": [
            {
                "config": {
                    "protocol": "http"
                },
                "kind": "proxy-defaults",
                "name": "global"
            },
            {
                "kind": "service-defaults",
                "name": "http-1",
                "protocol": "http"
            },
            {
                "kind": "service-defaults",
                "name": "grpc",
                "protocol": "grpc"
            },
            {
                "kind": "service-router",
                "name": "http-1",
                "routes": [

                ]
            },
            {
                "kind": "service-router",
                "name": "grpc",
                "routes": [

                ]
            }
        ]
    },
    "connect": {
        "ca_config": {
            "csr_max_concurrent": 2,
            "csr_max_per_second": 0,
            "leaf_cert_ttl": "72h"
        },
        "enabled": true
    },
    "data_dir": "/opt/consul",
    "datacenter": "datacenter",
    "default_intention_policy": "allow",
    "disable_remote_exec": true,
    "disable_update_check": true,
    "dns_config": {
        "allow_stale": true,
        "max_stale": "87600h",
        "service_ttl": {
            "*": "2s",
            "service_name": "5s",
            "service_name": "5s",
            "service_name": "5s",
            "service_name": "10s",
            "service_name": "2s",
            "service_name": "5s"
        }
    },
    "enable_central_service_config": true,
    "enable_script_checks": true,
    "enable_syslog": true,
    "leave_on_terminate": true,
    "log_acl_info": true,
    "log_level": "INFO",
    "node_meta": {
        "cluster": "cluster",
        "machine_type": "machine_type",
        "zone": "zone"
    },
    "node_name": "node_name",
    "ports": {
        "dns": 8600,
        "grpc": 8502,
        "grpc_tls": 8503,
        "http": 8500,
        "https": 8501,
        "serf_lan": 8301,
        "serf_wan": 8302
    },
    "primary_datacenter": "datacenter",
    "retry_join": [
        "provider=gce tag_value=tag_value zone_pattern=us-central1-.*"
    ],
    "retry_join_wan": [

    ],
    "server": true,
    "skip_leave_on_interrupt": false,
    "telemetry": {
        "disable_hostname": true,
        "prometheus_retention_time": "40s"
    },
    "tls": {
        "defaults": {
            "ca_file": "/etc/consul/ssl/ca.pem",
            "cert_file": "/etc/consul/ssl/cert.pem",
            "key_file": "/etc/consul/ssl/key.pem",
            "tls_min_version": "TLSv1_2",
            "verify_incoming": false,
            "verify_outgoing": false
        }
    },
    "ui": true,
    "use_streaming_backend": false
}

Operating system and Environment details

x86 debian 12, gcp

Log Fragments

Instance running consul connect envoy -sidecar-for=service-one:

agent.cache: handling error in Cache.Notify: cache-type=trust-bundles error="rpc error: code = Unknown desc = Permission denied: token with AccessorID '<redacted-default-token>' lacks permission 'service:write' on \"service-one\"" index=674388994

Instance running consul connect envoy -gateway=ingress

agent.client: RPC failed to server: method=ConnectCA.Sign server=<redacted ip>:8300 error="rpc error making call: rpc error making call: Permission denied: token with AccessorID '<redacted-default-token>' lacks permission 'service:write' on \"redacted-ingress-name\""