Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CRASM-2286:Add Regression Test Workflow with Dockerized Playwright in AWS #824

Open
wants to merge 29 commits into
base: develop
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
0d15da6
Add new integration test, integration.yml
JCantu248 Jan 22, 2025
7b7bab4
vulnerabilities tests
jyang2139 Jan 28, 2025
21cb495
fixed tests
jyang2139 Jan 28, 2025
fb459ed
put test update
jyang2139 Jan 29, 2025
6fe3611
test
jyang2139 Jan 29, 2025
41a384c
Add method to return domain, vulnerability uids for other test cases.
JCantu248 Jan 30, 2025
15c37a7
updated test for new methods
jyang2139 Jan 30, 2025
f6ab796
Change get_domains, get_vulnerabilities to return entire object, not …
JCantu248 Feb 3, 2025
76d2a92
updates to id method
jyang2139 Feb 3, 2025
ebd4add
fixing environment variable
jyang2139 Feb 4, 2025
a083545
Merge pull request #774 from cisagov/jy_integration_tests
JCantu248 Feb 4, 2025
3df9356
Code cleanup for test_vulnerabilities.py.
JCantu248 Feb 5, 2025
dc09d63
Testing integration workflow on push.
JCantu248 Feb 6, 2025
bfd732a
Add requirements.txt, changes to test_domains.
JCantu248 Feb 6, 2025
17db7c3
Update integration.yml
aloftus23 Feb 6, 2025
c8454f6
Reverting integration.yml to only on deployment status.
JCantu248 Feb 6, 2025
6ba373a
Merge branch 'JD_integration_tests' of github.com:cisagov/XFD into JD…
JCantu248 Feb 6, 2025
45b81c9
Merge branch 'develop' into JD_integration_tests
JCantu248 Feb 26, 2025
6012cd3
Merge branch 'develop' of github.com:cisagov/XFD into JD_integration_…
JCantu248 Mar 11, 2025
abb21c2
Merge branch 'develop' of github.com:cisagov/XFD into develop
JCantu248 Mar 19, 2025
8e8f3eb
Merge branch 'develop' of github.com:cisagov/XFD into develop
JCantu248 Mar 21, 2025
2a42ebd
Add regression.yml for Dockerized Playwright actions, modifications t…
JCantu248 Mar 21, 2025
d1459d3
Remove integration tests that were accidentally included in this branch
JCantu248 Mar 21, 2025
b31871d
Fixed startup issue with Github secrets reference in container image.
JCantu248 Mar 21, 2025
30225d6
Create separate jobs for playwright tests on develop, integration bra…
JCantu248 Mar 27, 2025
a53fabb
Merge branch 'develop' of github.com:cisagov/XFD into CRASM_2286_Gith…
JCantu248 Mar 27, 2025
ada20d8
Add quotes for container image in playwright terraform config.
JCantu248 Mar 27, 2025
4c5403e
Add a comma after image.
JCantu248 Mar 27, 2025
12872c9
Fix yaml linter errors with line length on regression.yml
JCantu248 Mar 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixed tests
jyang2139 committed Jan 28, 2025
commit 21cb49514c874760625799c52b88c5d0a36cf790
112 changes: 78 additions & 34 deletions integration/tests/test_vulnerabilities.py
Original file line number Diff line number Diff line change
@@ -12,31 +12,41 @@
VULNERABILITIES_ID = os.environ.get("VULNERABILITIES_ID")
BAD_ID = "c0effe93-3647-475a-a0c5-0b629c348590"


# mark tests with integration tag, run with pytest -m integration
@pytest.mark.integration
def test_get_vulnerability_by_id():
"""Test get vulnerability by ID."""
url = f"{BASE_URL}/vulnerabilities/{VULNERABILITIES_ID}"
response = requests.get(url, headers={"X-API-KEY": X_API_KEY}, timeout=10)

assert response.status_code == 200, f"Expected status 200, got {response.status_code}"
assert (
response.status_code == 200
), f"Expected status 200, got {response.status_code}"
data = response.json()
assert data is not None, "Response is empty"
assert data["id"] == VULNERABILITIES_ID, f"Expected ID {VULNERABILITIES_ID}, got {data['id']}"
assert (
data["id"] == VULNERABILITIES_ID
), f"Expected ID {VULNERABILITIES_ID}, got {data['id']}"


@pytest.mark.integration
def test_get_vulnerability_by_id_fails_404():
"""Test get vulnerability by ID fails with 404."""
url = f"{BASE_URL}/vulnerabilities/{BAD_ID}"
response = requests.get(url, headers={"X-API-KEY": X_API_KEY}, timeout=10)

assert response.status_code == 404, f"Expected status 404, got {response.status_code}"

assert (
response.status_code == 404
), f"Expected status 404, got {response.status_code}"
data = response.json()
assert data is not None, "Response is empty"

# Check for the error message in the "detail" key
assert "detail" in data, "Expected 'detail' in response"
assert data["detail"] == "Vulnerability not found.", f"Unexpected error message: {data['detail']}"
assert (
data["detail"] == "Vulnerability not found."
), f"Unexpected error message: {data['detail']}"


@pytest.mark.integration
@@ -54,15 +64,19 @@ def test_search_vulnerabilities():
url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10
)

assert response.status_code == 200, f"Expected status 200, got {response.status_code}"
assert (
response.status_code == 200
), f"Expected status 200, got {response.status_code}"
data = response.json()
assert data is not None, "Response is empty"
assert "result" in data, "Results not found in response"
assert len(data["result"]) > 0, "No results found"

# Validate that results include vulnerabilities with high severity (case-insensitive)
for vulnerability in data["result"]:
assert vulnerability["severity"].lower() == "high", f"Expected severity 'high', got {vulnerability['severity']}"
assert (
vulnerability["severity"].lower() == "high"
), f"Expected severity 'high', got {vulnerability['severity']}"


@pytest.mark.integration
@@ -84,7 +98,7 @@ def test_update_vulnerability_by_id():
"url": "https://updated-url.com",
"name": "Updated Reference",
"tags": ["Updated Tag"],
"source": "UPDATED_SOURCE"
"source": "UPDATED_SOURCE",
}
],
"cvss": 7.5,
@@ -100,25 +114,40 @@ def test_update_vulnerability_by_id():
"domain_id": "0c4ee5b6-ff18-458c-adcc-dfe121fb54c5",
"service_id": "9ac326f0-29ad-4e2c-a6bf-e330c91aa872",
}
response = requests.put(url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10)
response = requests.put(
url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10
)

assert response.status_code == 200, f"Expected status 200, got {response.status_code}"
assert (
response.status_code == 200
), f"Expected status 200, got {response.status_code}"
data = response.json()

# Validate updated fields
assert data["title"] == "Updated CVE-2019-6109", "Title was not updated correctly"
assert data["description"] == "Updated description for this vulnerability.", "Description was not updated correctly"
assert (
data["description"] == "Updated description for this vulnerability."
), "Description was not updated correctly"
assert data["severity"] == "High", "Severity was not updated correctly"
assert data["state"] == "closed", "State was not updated correctly"

# Validate references
assert len(data["references"]) == 1, "References were not updated correctly"
assert data["references"][0]["url"] == "https://updated-url.com", "Reference URL was not updated correctly"
assert data["references"][0]["source"] == "UPDATED_SOURCE", "Reference source was not updated correctly"
assert (
data["references"][0]["url"] == "https://updated-url.com"
), "Reference URL was not updated correctly"
assert (
data["references"][0]["source"] == "UPDATED_SOURCE"
), "Reference source was not updated correctly"

assert data.get("domain_id") == "0c4ee5b6-ff18-458c-adcc-dfe121fb54c5", "Domain ID mismatch"
assert (
data.get("domain_id") == "0c4ee5b6-ff18-458c-adcc-dfe121fb54c5"
), "Domain ID mismatch"
# Validate service_id
assert data["service_id"] == "9ac326f0-29ad-4e2c-a6bf-e330c91aa872", "Service ID mismatch"
assert (
data["service_id"] == "9ac326f0-29ad-4e2c-a6bf-e330c91aa872"
), "Service ID mismatch"


@pytest.mark.integration
def test_update_vulnerability_by_id_fails_404():
@@ -145,16 +174,21 @@ def test_update_vulnerability_by_id_fails_404():
"actions": [],
"structuredData": {},
"isKev": False,
"domain_id": "0c4ee5b6-ff18-458c-adcc-dfe121fb54c5",
"service_id": "9ac326f0-29ad-4e2c-a6bf-e330c91aa872",
"domain_id": "0c4ee5b6-ff18-458c-adcc-dfe121fb54c5",
"service_id": "9ac326f0-29ad-4e2c-a6bf-e330c91aa872",
}
response = requests.put(url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10)
response = requests.put(
url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10
)

assert response.status_code == 404, f"Expected status 404, got {response.status_code}"
assert (
response.status_code == 404
), f"Expected status 404, got {response.status_code}"
data = response.json()
assert "detail" in data, "Error detail missing in response"
assert data["detail"] == "Vulnerability not found.", "Unexpected error message"


@pytest.mark.integration
def test_update_vulnerability_by_id_fails_422():
"""Test update vulnerability by ID fails with 422 due to invalid payload."""
@@ -183,31 +217,41 @@ def test_update_vulnerability_by_id_fails_422():
"domain_id": "invalid-domain-id", # Invalid domain_id
"service_id": "invalid-service-id", # Invalid service_id
}
response = requests.put(url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10)
response = requests.put(
url, json=json, headers={"X-API-KEY": X_API_KEY}, timeout=10
)

assert response.status_code == 422, f"Expected status 422, got {response.status_code}"
assert (
response.status_code == 422
), f"Expected status 422, got {response.status_code}"
data = response.json()

# Debugging: Print the response
print(data) # Inspect the response to debug validation errors

# Validate that the response contains a list of validation errors
assert isinstance(data["detail"], list), "Expected 'detail' to be a list of validation errors"
assert isinstance(
data["detail"], list
), "Expected 'detail' to be a list of validation errors"

# Check that 'createdAt' error is included
created_at_error = next((error for error in data["detail"] if "createdAt" in error.get("loc", [])), None)
created_at_error = next(
(error for error in data["detail"] if "createdAt" in error.get("loc", [])), None
)
assert created_at_error is not None, "'createdAt' validation error is missing"
assert "Input should be a valid datetime" in created_at_error["msg"], \
f"Unexpected error message: {created_at_error['msg']}"
assert (
"Input should be a valid datetime" in created_at_error["msg"]
), f"Unexpected error message: {created_at_error['msg']}"

# Check that 'title' error is included (optional warning if missing)
title_error = next((error for error in data["detail"] if "title" in error.get("loc", [])), None)
title_error = next(
(error for error in data["detail"] if "title" in error.get("loc", [])), None
)
if title_error is None:
print("Warning: 'title' validation error is missing. Validation might not be implemented.")
print(
"Warning: 'title' validation error is missing. Validation might not be implemented."
)
else:
assert title_error["msg"] == "Field required", f"Unexpected error message: {title_error['msg']}"





assert (
title_error["msg"] == "Field required"
), f"Unexpected error message: {title_error['msg']}"