feat: support set ssl_trusted_store for upstream (#95)

AlinsRan · web-flow · commit 6658589e6c70 · 2025-01-20T08:16:31.000+08:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,6 +30,7 @@ jobs:
         run: |
           sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl luarocks libpcre3 libpcre3-dev zlib1g-dev
           sudo luarocks install lua-resty-http > build.log 2>&1 || (cat build.log && exit 1)
+          sudo luarocks install lua-resty-openssl > build.log 2>&1 || (cat build.log && exit 1)
 
       - name: Before install
         run: |
diff --git a/lib/resty/apisix/upstream.lua b/lib/resty/apisix/upstream.lua
@@ -1,17 +1,19 @@
 local ffi = require("ffi")
 local base = require("resty.core.base")
 local get_request = base.get_request
+local get_phase = ngx.get_phase
 local C = ffi.C
 local NGX_ERROR = ngx.ERROR
+local NGX_OK = ngx.OK
 
 
 base.allows_subsystem("http")
 
 
 ffi.cdef([[
 typedef intptr_t        ngx_int_t;
-ngx_int_t
-ngx_http_apisix_upstream_set_cert_and_key(ngx_http_request_t *r, void *cert, void *key);
+ngx_int_t ngx_http_apisix_upstream_set_cert_and_key(ngx_http_request_t *r, void *cert, void *key);
+ngx_int_t ngx_http_apisix_upstream_set_ssl_trusted_store(ngx_http_request_t *r, void *store);
 ]])
 local _M = {}
 
@@ -30,5 +32,42 @@ function _M.set_cert_and_key(cert, key)
     return true
 end
 
+local set_ssl_trusted_store
+do
+    local ALLOWED_PHASES = {
+        ['rewrite'] = true,
+        ['balancer'] = true,
+        ['access'] = true,
+        ['preread'] = true,
+    }
+
+    local openssl_x509_store = require "resty.openssl.x509.store"
+    function set_ssl_trusted_store(store)
+        if not ALLOWED_PHASES[get_phase()] then
+            error("API disabled in the current context", 2)
+        end
+
+        if not openssl_x509_store.istype(store) then
+            error("store expects a resty.openssl.x509.store" ..
+                " object but get " .. type(store), 2)
+        end
+
+        local r = get_request()
+
+        local ret = C.ngx_http_apisix_upstream_set_ssl_trusted_store(
+            r, store.ctx)
+        if ret == NGX_OK then
+            return true
+        end
+
+        if ret == NGX_ERROR then
+            return nil, "error while setting upstream trusted store"
+        end
+
+        error("unknown return code: " .. tostring(ret))
+    end
+end
+_M.set_ssl_trusted_store = set_ssl_trusted_store
+
 
 return _M
diff --git a/src/ngx_http_apisix_module.c b/src/ngx_http_apisix_module.c
@@ -158,6 +158,15 @@ ngx_http_apisix_cleanup_cert_and_key(ngx_http_apisix_ctx_t *ctx)
         ctx->upstream_pkey = NULL;
     }
 }
+
+static void
+ngx_http_apisix_cleanup_trusted_store(ngx_http_apisix_ctx_t *ctx)
+{
+    if (ctx->upstream_trusted_store != NULL) {
+        X509_STORE_free(ctx->upstream_trusted_store);
+        ctx->upstream_trusted_store = NULL;
+    }
+}
 #endif
 
 
@@ -168,6 +177,7 @@ ngx_http_apisix_cleanup(void *data)
 
 #if (NGX_HTTP_SSL)
     ngx_http_apisix_cleanup_cert_and_key(ctx);
+    ngx_http_apisix_cleanup_trusted_store(ctx);
 #endif
 }
 
@@ -250,6 +260,41 @@ ngx_http_apisix_upstream_set_cert_and_key(ngx_http_request_t *r,
     return NGX_ERROR;
 }
 
+ngx_int_t
+ngx_http_apisix_upstream_set_ssl_trusted_store(ngx_http_request_t *r, void *data)
+{
+    X509_STORE                  *store = data;
+    ngx_http_apisix_ctx_t       *ctx;
+
+    if (store == NULL) {
+        return NGX_ERROR;
+    }
+
+    ctx = ngx_http_apisix_get_module_ctx(r);
+
+    if (ctx == NULL) {
+        return NGX_ERROR;
+    }
+
+    if (ctx->upstream_trusted_store != NULL) {
+        ngx_http_apisix_cleanup_trusted_store(ctx);
+    }
+    
+    if (X509_STORE_up_ref(store) == 0) {
+        goto failed;
+    }
+
+    ctx->upstream_trusted_store = store;
+
+    return NGX_OK;
+
+failed:
+
+    ngx_http_apisix_flush_ssl_error();
+
+    return NGX_ERROR;
+}
+
 
 void
 ngx_http_apisix_set_upstream_ssl(ngx_http_request_t *r, ngx_connection_t *c)
@@ -258,6 +303,7 @@ ngx_http_apisix_set_upstream_ssl(ngx_http_request_t *r, ngx_connection_t *c)
     ngx_http_apisix_ctx_t       *ctx;
     STACK_OF(X509)              *cert;
     EVP_PKEY                    *pkey;
+    X509_STORE                  *store;
     X509                        *x509;
 #ifdef OPENSSL_IS_BORINGSSL
     size_t                       i;
@@ -275,8 +321,9 @@ ngx_http_apisix_set_upstream_ssl(ngx_http_request_t *r, ngx_connection_t *c)
     }
 
     if (ctx->upstream_cert != NULL) {
-        cert = ctx->upstream_cert;
-        pkey = ctx->upstream_pkey;
+        cert  = ctx->upstream_cert;
+        pkey  = ctx->upstream_pkey;
+        store = ctx->upstream_trusted_store;
 
         if (sk_X509_num(cert) < 1) {
             ngx_ssl_error(NGX_LOG_ERR, c->log, 0,
@@ -317,6 +364,17 @@ ngx_http_apisix_set_upstream_ssl(ngx_http_request_t *r, ngx_connection_t *c)
                           "SSL_use_PrivateKey() failed");
             goto failed;
         }
+
+        if (store != NULL) {
+            ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                           "overriding upstream SSL trusted store");
+        
+            if (SSL_set1_verify_cert_store(sc, store) == 0) {
+                ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                              "SSL_set1_verify_cert_store() failed");
+                goto failed;
+            }
+        }
     }
 
     return;
diff --git a/src/ngx_http_apisix_module.h b/src/ngx_http_apisix_module.h
@@ -22,6 +22,7 @@ typedef struct {
 typedef struct {
     STACK_OF(X509)      *upstream_cert;
     EVP_PKEY            *upstream_pkey;
+    X509_STORE          *upstream_trusted_store;
 
     off_t                client_max_body_size;
 
diff --git a/t/upstream_mtls2.t b/t/upstream_mtls2.t
@@ -0,0 +1,131 @@
+use t::APISIX_NGINX 'no_plan';
+
+repeat_each(2);
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!$block->http_config) {
+        my $http_config = <<'_EOC_';
+    server {
+        listen unix:$TEST_NGINX_HTML_DIR/nginx.sock ssl;
+        server_name admin.apisix.dev;
+        ssl_certificate ../../certs/mtls_server.crt;
+        ssl_certificate_key ../../certs/mtls_server.key;
+        ssl_client_certificate ../../certs/mtls_ca.crt;
+        ssl_verify_client on;
+
+        server_tokens off;
+
+        location /foo {
+            return 200 'ok\n';
+        }
+    }
+
+_EOC_
+
+        $block->set_value("http_config", $http_config);
+    }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: only set client certificate
+--- config
+    location /t {
+        access_by_lua_block {
+            local upstream = require("resty.apisix.upstream")
+            local ssl = require("ngx.ssl")
+
+            local f = assert(io.open("t/certs/mtls_client.crt"))
+            local cert_data = f:read("*a")
+            f:close()
+
+            local cert = assert(ssl.parse_pem_cert(cert_data))
+
+            f = assert(io.open("t/certs/mtls_client.key"))
+            local key_data = f:read("*a")
+            f:close()
+
+            local key = assert(ssl.parse_pem_priv_key(key_data))
+
+            local ok, err = upstream.set_cert_and_key(cert, key)
+            if not ok then
+                ngx.say("set_cert_and_key failed: ", err)
+            end
+        }
+
+        proxy_ssl_trusted_certificate ../../certs/mtls_client.crt;
+        proxy_ssl_verify on;
+        proxy_ssl_name admin.apisix.dev;
+        proxy_pass https://unix:$TEST_NGINX_HTML_DIR/nginx.sock:/foo;
+    }
+--- error_code: 502
+--- error_log
+unable to verify the first certificate
+
+
+
+=== TEST 2: send valid client certificate
+--- config
+    location /t {
+        access_by_lua_block {
+            local upstream = require("resty.apisix.upstream")
+            local ssl = require("ngx.ssl")
+
+            local f = assert(io.open("t/certs/mtls_client.crt"))
+            local cert_data = f:read("*a")
+            f:close()
+
+            local cert = assert(ssl.parse_pem_cert(cert_data))
+
+            f = assert(io.open("t/certs/mtls_client.key"))
+            local key_data = f:read("*a")
+            f:close()
+
+            local key = assert(ssl.parse_pem_priv_key(key_data))
+
+            local ok, err = upstream.set_cert_and_key(cert, key)
+            if not ok then
+                ngx.say("set_cert_and_key failed: ", err)
+            end
+
+            f = assert(io.open("t/certs/mtls_ca.crt"))
+            local ca_data = f:read("*a")
+            f:close()
+
+            local ca_cert = assert(ssl.parse_pem_cert(ca_data))
+
+            local openssl_x509_store = require "resty.openssl.x509.store"
+            local openssl_x509 = require "resty.openssl.x509"
+            local trust_store, err = openssl_x509_store.new()
+            if err then
+                ngx.log(ngx.ERR, "failed to create trust store: ", err)
+                ngx.exit(500)
+            end
+
+            local x509, err = openssl_x509.new(ca_data, "PEM")
+
+            local _, err = trust_store:add(x509)
+            if err then
+                ngx.log(ngx.ERR, "failed to add ca cert to trust store: ", err)
+                ngx.exit(500)
+            end
+
+            local ok, err = upstream.set_ssl_trusted_store(trust_store)
+            if not ok then
+                ngx.log(ngx.ERR, "set_ssl_trusted_store failed: ", err)
+                ngx.exit(500)
+            end
+        }
+
+        proxy_ssl_trusted_certificate ../../certs/mtls_client.crt;
+        proxy_ssl_verify on;
+        proxy_ssl_name admin.apisix.dev;
+        proxy_pass https://unix:$TEST_NGINX_HTML_DIR/nginx.sock:/foo;
+    }
+
+--- response_body
+ok
