Skip to content

Commit

Permalink
nginx/appsec support
Browse files Browse the repository at this point in the history
  • Loading branch information
cataphract committed Jan 17, 2025
1 parent 56e10a4 commit f0763ac
Show file tree
Hide file tree
Showing 16 changed files with 723 additions and 84 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ jobs:
uses: actions/checkout@v4
- name: Get library artifact
run: ./utils/scripts/load-binary.sh ${{ matrix.library }}
- name: Get nginx module
if: matrix.library == 'cpp'
run: ./utils/scripts/load-binary.sh nginx
env:
CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }}

- name: Get agent artifact
run: ./utils/scripts/load-binary.sh agent
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/run-end-to-end.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,15 +196,15 @@ jobs:
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, 'AGENT_NOT_SUPPORTING_SPAN_EVENTS') && (inputs.library != 'ruby' || matrix.weblog == 'rack')
run: ./run.sh AGENT_NOT_SUPPORTING_SPAN_EVENTS
- name: Run APPSEC_MISSING_RULES scenario
# C++ 1.2.0 freeze when the rules file is missing
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_MISSING_RULES"') && inputs.library != 'cpp'
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_MISSING_RULES"') && matrix.weblog != 'nginx'
# nginx 1.2.0 refuses to start without a valid rules files
run: ./run.sh APPSEC_MISSING_RULES
- name: Run APPSEC_CUSTOM_RULES scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CUSTOM_RULES"')
run: ./run.sh APPSEC_CUSTOM_RULES
- name: Run APPSEC_CORRUPTED_RULES scenario
# C++ 1.2.0 freeze when the rules file is missing
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CORRUPTED_RULES"') && inputs.library != 'cpp'
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_CORRUPTED_RULES"') && matrix.weblog != 'nginx'
# nginx 1.2.0 refuses to start without a valid rules files
run: ./run.sh APPSEC_CORRUPTED_RULES
- name: Run APPSEC_RULES_MONITORING_WITH_ERRORS scenario
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_RULES_MONITORING_WITH_ERRORS"')
Expand Down
329 changes: 303 additions & 26 deletions manifests/cpp.yml

Large diffs are not rendered by default.

11 changes: 10 additions & 1 deletion tests/appsec/test_blocking_addresses.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# Copyright 2021 Datadog, Inc.

import json

from utils import (
bug,
context,
Expand Down Expand Up @@ -56,6 +57,7 @@ def test_blocking(self):
def setup_blocking_before(self):
self.block_req2 = weblog.get("/tag_value/tainted_value_6512/200", headers={"X-Forwarded-For": "1.1.1.1"})

@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# second request should block and must not set the tag in span
Expand Down Expand Up @@ -114,6 +116,7 @@ def setup_blocking_before(self):
context.scenario is scenarios.external_processing_blocking,
reason="The endpoint /tag_value is not implemented in the weblog",
)
@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down Expand Up @@ -172,6 +175,7 @@ def setup_blocking_before(self):
context.scenario is scenarios.external_processing_blocking,
reason="The endpoint /tag_value is not implemented in the weblog",
)
@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down Expand Up @@ -230,6 +234,7 @@ def setup_blocking_before(self):
context.scenario is scenarios.external_processing_blocking,
reason="The endpoint /param is not implemented in the weblog",
)
@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down Expand Up @@ -283,6 +288,7 @@ def setup_blocking_before(self):
context.scenario is scenarios.external_processing_blocking,
reason="The endpoint /tag_value is not implemented in the weblog",
)
@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down Expand Up @@ -336,6 +342,7 @@ def setup_blocking_before(self):
context.scenario is scenarios.external_processing_blocking,
reason="The endpoint /tag_value is not implemented in the weblog",
)
@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down Expand Up @@ -389,6 +396,7 @@ def setup_blocking_before(self):
context.scenario is scenarios.external_processing_blocking,
reason="The endpoint /tag_value is not implemented in the weblog",
)
@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down Expand Up @@ -440,7 +448,7 @@ def setup_non_blocking_plain_text(self):
)

@irrelevant(
context.weblog_variant in ("akka-http", "play", "jersey-grizzly2", "resteasy-netty3"),
context.weblog_variant in ("akka-http", "play", "jersey-grizzly2", "resteasy-netty3", "nginx"),
reason="Blocks on text/plain if parsed to a String",
)
def test_non_blocking_plain_text(self):
Expand All @@ -453,6 +461,7 @@ def setup_blocking_before(self):
self.set_req1 = weblog.post("/tag_value/clean_value_3882/200", data={"good": "value"})
self.block_req2 = weblog.post("/tag_value/tainted_value_body/200", data={"value5": "bsldhkuqwgervf"})

@irrelevant(context.weblog_variant == "nginx", reason="Tag adding happens before WAF run")
def test_blocking_before(self):
"""Test that blocked requests are blocked before being processed"""
# first request should not block and must set the tag in span accordingly
Expand Down
2 changes: 1 addition & 1 deletion tests/appsec/test_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from utils.tools import nested_lookup


RUNTIME_FAMILIES = ["nodejs", "ruby", "jvm", "dotnet", "go", "php", "python"]
RUNTIME_FAMILIES = ["nodejs", "ruby", "jvm", "dotnet", "go", "php", "python", "cpp"]


@bug(context.library == "[email protected]", reason="APMRP-360")
Expand Down
1 change: 1 addition & 0 deletions tests/appsec/waf/test_addresses.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ def setup_specific_key2(self):

@irrelevant(library="ruby", reason="Rack transforms underscores into dashes")
@irrelevant(library="php", reason="PHP normalizes into dashes; additionally, matching on keys is not supported")
@irrelevant(weblog_variant="nginx", reason="Header rejected by nginx ('client sent invalid header line'")
@missing_feature(weblog_variant="spring-boot-3-native", reason="GraalVM. Tracing support only")
def test_specific_key2(self):
"""attacks on specific header X_Filename, and report it"""
Expand Down
9 changes: 7 additions & 2 deletions utils/_context/containers.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ def start(self, network: Network) -> Container:

logger.info(f"Start container {self.container_name}")

# the whole thing is reimplemented in python...
if self.healthcheck is not None:
self.kwargs["healthcheck"] = {"test": ["NONE"]}

self._container = _get_client().containers.run(
image=self.image.name,
name=self.container_name,
Expand Down Expand Up @@ -997,9 +1001,10 @@ def __init__(self, host_log_folder) -> None:
class MySqlContainer(SqlDbTestedContainer):
def __init__(self, host_log_folder) -> None:
super().__init__(
image_name="mysql/mysql-server:latest",
image_name="mysql/mysql-server:8.0.32",
name="mysqldb",
command="--default-authentication-plugin=mysql_native_password",
command="--lc-messages-dir=/usr/share/mysql-8.0/english "
"--default-authentication-plugin=mysql_native_password",
environment={
"MYSQL_DATABASE": "mysql_dbname",
"MYSQL_USER": "mysqldb",
Expand Down
14 changes: 11 additions & 3 deletions utils/build/docker/cpp/nginx.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,24 @@ ARG NGINX_VERSION="1.25.4"
ENV NGINX_VERSION=${NGINX_VERSION}

RUN apt-get update \
&& apt-get install -y wget tar jq curl xz-utils stress-ng binutils
&& apt-get install -y \
wget tar jq curl xz-utils stress-ng binutils gcc libmicrohttpd-dev \
procps gdb

RUN mkdir /builds
RUN mkdir /builds /binaries

COPY utils/build/docker/cpp/nginx/nginx.conf /etc/nginx/nginx.conf
COPY utils/build/docker/cpp/nginx/nginx.conf /etc/nginx/nginx.conf.no-waf
COPY utils/build/docker/cpp/nginx/nginx-waf.conf /etc/nginx/nginx.conf.waf
COPY utils/build/docker/cpp/nginx/hello.html /builds/hello.html
COPY utils/build/docker/cpp/nginx/install_ddtrace.sh /builds/
COPY utils/build/docker/cpp/install_ddprof.sh /builds/
COPY utils/build/docker/cpp/nginx/app.sh /builds/
COPY utils/build/docker/cpp/ binaries* /builds/
COPY binaries/* /binaries/
COPY utils/build/docker/cpp/nginx/backend.c /tmp/

# install backend app
RUN gcc -O0 -g -o /usr/local/bin/backend /tmp/backend.c -lmicrohttpd && rm /tmp/backend.c

WORKDIR /builds

Expand Down
2 changes: 2 additions & 0 deletions utils/build/docker/cpp/nginx/app.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/bash

backend &

if [[ "${DDPROF_ENABLE:-,,}" == "yes" ]]; then
ddprof -l notice nginx -g 'daemon off;'
else
Expand Down
114 changes: 114 additions & 0 deletions utils/build/docker/cpp/nginx/backend.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#include <fcntl.h>
#include <microhttpd.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <stdio.h>
#include <sys/stat.h>

#define PORT 7778

static struct MHD_Daemon *daemon_;

static ssize_t read_data(void *cls, uint64_t pos, char *buf, size_t max) {
int fd = (int)(uintptr_t)cls;
ssize_t num_read = read(fd, buf, max);
if (num_read == 0) {
return -1;
}
if (num_read < 0) {
return -2;
}
return num_read;
}
static void close_fd(void *cls) {
close((int)(uintptr_t)cls);
}

static enum MHD_Result answer_to_connection(void *cls, struct MHD_Connection *connection,
const char *url, const char *method,
const char *version, const char *upload_data,
size_t *upload_data_size, void **con_cls)
{
const char *status_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "status");
const char *value = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "value");

if (strcmp(url, "/read_file") == 0) {
const char *val = MHD_lookup_connection_value (connection, MHD_GET_ARGUMENT_KIND, "file");
if (!val) {
return MHD_NO;
}

const int fd = open(val, O_RDONLY);
if (fd < 0) {
return MHD_NO;
}

struct stat st;
if (fstat(fd, &st) != 0) {
close(fd);
return MHD_NO;
}
const size_t file_size = st.st_size;

struct MHD_Response *response;
if (file_size == 0) {
response = MHD_create_response_from_callback(-1, 512, read_data, (void*)(uintptr_t)fd, close_fd);
} else {
response = MHD_create_response_from_fd((uint64_t)file_size, fd);
}

MHD_add_response_header(response, "Content-Type", "application/octet-stream");
int ret = MHD_queue_response(connection, 200, response);
MHD_destroy_response(response);
return ret;
}

if (strcmp(url, "/content") != 0 || !status_str || !value)
return MHD_NO; // Only respond to the correct URL and if all parameters are present

int status_code = atoi(status_str);
if (status_code <= 0)
return MHD_NO; // Ensure the status code is a valid positive integer

const char *page = value;
struct MHD_Response *response = MHD_create_response_from_buffer(strlen(page), (void*)page, MHD_RESPMEM_MUST_COPY);
MHD_add_response_header(response, "Content-Type", "text/html");

int ret = MHD_queue_response(connection, status_code, response);
MHD_destroy_response(response);
return ret;
}

void stop_server(int signum)
{
if (daemon_)
{
MHD_stop_daemon(daemon_);
daemon_ = NULL;
printf("Server has been stopped.\n");
}
exit(0);
}

int main()
{
struct sigaction action;
memset(&action, 0, sizeof(struct sigaction));
action.sa_handler = stop_server;
sigaction(SIGINT, &action, NULL); // Handle SIGINT
sigaction(SIGTERM, &action, NULL); // Handle SIGTERM

daemon_ = MHD_start_daemon(MHD_USE_INTERNAL_POLLING_THREAD, PORT, NULL, NULL,
&answer_to_connection, NULL, MHD_OPTION_END);
if (!daemon_)
{
fprintf(stderr, "Failed to start the daemon.\n");
return 1;
}

printf("Server running on port %d\n", PORT);
pause(); // Wait for signals

return 0;
}
Loading

0 comments on commit f0763ac

Please sign in to comment.