Skip to content

Commit 1232214

Browse files
scottHATscottHAT
committed
expanding test coverage
1 parent 35c396c commit 1232214

19 files changed

+3978
-272
lines changed

‎.coverage

16 KB
Binary file not shown.

‎README.md

+103-199
Large diffs are not rendered by default.

‎RELEASE_NOTES.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# MCP Protocol Validator v1.0.0
1+
# MCP Protocol Validator v0.0.1
22

33
## Initial Release
44

‎SECURITY.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ We currently provide security updates for the following versions of the MCP Prot
66

77
| Version | Supported |
88
| ------- | ------------------ |
9-
| 1.0.x | :white_check_mark: |
9+
| 0.1.0 | :white_check_mark: |
1010

1111
## Reporting a Vulnerability
1212

‎mcp_testing/scripts/http_test.py

+77-70
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import argparse
1111
import os
1212
import sys
13+
import traceback
1314
from pathlib import Path
1415

1516
# Add the project root to the Python path
@@ -20,80 +21,86 @@
2021

2122
def main():
2223
"""Run compliance tests against an HTTP MCP server."""
23-
parser = argparse.ArgumentParser(description="Run tests against an HTTP MCP server")
24-
parser.add_argument(
25-
"--server-url",
26-
default="http://localhost:9000/mcp",
27-
help="URL of the MCP HTTP server (default: http://localhost:9000/mcp)"
28-
)
29-
parser.add_argument(
30-
"--protocol-version",
31-
choices=["2024-11-05", "2025-03-26"],
32-
default="2025-03-26",
33-
help="Protocol version to use (default: 2025-03-26)"
34-
)
35-
parser.add_argument(
36-
"--debug",
37-
action="store_true",
38-
help="Enable debug logging"
39-
)
40-
parser.add_argument(
41-
"--output-dir",
42-
help="Directory to write test results"
43-
)
44-
parser.add_argument(
45-
"--max-retries",
46-
type=int,
47-
default=3,
48-
help="Maximum number of connection retries"
49-
)
50-
parser.add_argument(
51-
"--retry-interval",
52-
type=int,
53-
default=2,
54-
help="Seconds to wait between connection retries"
55-
)
56-
57-
args = parser.parse_args()
58-
59-
# Create output directory if needed
60-
if args.output_dir:
61-
os.makedirs(args.output_dir, exist_ok=True)
62-
63-
# Check server connection first
64-
if not wait_for_server(
65-
args.server_url,
66-
max_retries=args.max_retries,
67-
retry_interval=args.retry_interval
68-
):
69-
return 1
70-
71-
# Run the HTTP tests
72-
tester = MCPHttpTester(args.server_url, args.debug)
73-
tester.protocol_version = args.protocol_version
74-
75-
success = tester.run_all_tests()
76-
77-
# Generate report if needed
78-
if args.output_dir:
79-
report_path = os.path.join(args.output_dir, f"http_test_report_{args.protocol_version}.md")
24+
try:
25+
parser = argparse.ArgumentParser(description="Run tests against an HTTP MCP server")
26+
parser.add_argument(
27+
"--server-url",
28+
default="http://localhost:9000/mcp",
29+
help="URL of the MCP HTTP server (default: http://localhost:9000/mcp)"
30+
)
31+
parser.add_argument(
32+
"--protocol-version",
33+
choices=["2024-11-05", "2025-03-26"],
34+
default="2025-03-26",
35+
help="Protocol version to use (default: 2025-03-26)"
36+
)
37+
parser.add_argument(
38+
"--debug",
39+
action="store_true",
40+
help="Enable debug logging"
41+
)
42+
parser.add_argument(
43+
"--output-dir",
44+
help="Directory to write test results"
45+
)
46+
parser.add_argument(
47+
"--max-retries",
48+
type=int,
49+
default=3,
50+
help="Maximum number of connection retries"
51+
)
52+
parser.add_argument(
53+
"--retry-interval",
54+
type=int,
55+
default=2,
56+
help="Seconds to wait between connection retries"
57+
)
58+
59+
args = parser.parse_args()
60+
61+
# Create output directory if needed
62+
if args.output_dir:
63+
os.makedirs(args.output_dir, exist_ok=True)
64+
65+
# Check server connection first
66+
if not wait_for_server(
67+
args.server_url,
68+
max_retries=args.max_retries,
69+
retry_interval=args.retry_interval
70+
):
71+
return 1
8072

81-
with open(report_path, "w") as f:
82-
f.write(f"# MCP HTTP Compliance Test Report\n\n")
83-
f.write(f"- Server: {args.server_url}\n")
84-
f.write(f"- Protocol Version: {args.protocol_version}\n")
85-
f.write(f"- Date: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
73+
# Run the HTTP tests
74+
tester = MCPHttpTester(args.server_url, args.debug)
75+
tester.protocol_version = args.protocol_version
76+
77+
success = tester.run_all_tests()
78+
79+
# Generate report if needed
80+
if args.output_dir:
81+
report_path = os.path.join(args.output_dir, f"http_test_report_{args.protocol_version}.md")
8682

87-
f.write(f"## Test Results\n\n")
88-
f.write(f"All tests {'PASSED' if success else 'FAILED'}\n\n")
83+
with open(report_path, "w") as f:
84+
f.write(f"# MCP HTTP Compliance Test Report\n\n")
85+
f.write(f"- Server: {args.server_url}\n")
86+
f.write(f"- Protocol Version: {args.protocol_version}\n")
87+
f.write(f"- Date: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
88+
89+
f.write(f"## Test Results\n\n")
90+
f.write(f"All tests {'PASSED' if success else 'FAILED'}\n\n")
91+
92+
f.write(f"## Notes\n\n")
93+
f.write(f"This report was generated using the MCP HTTP testing framework.\n")
94+
f.write(f"For more detailed test results, run with the --debug flag.\n")
8995

90-
f.write(f"## Notes\n\n")
91-
f.write(f"This report was generated using the MCP HTTP testing framework.\n")
92-
f.write(f"For more detailed test results, run with the --debug flag.\n")
96+
print(f"Test report written to {report_path}")
9397

94-
print(f"Test report written to {report_path}")
95-
96-
return 0 if success else 1
98+
return 0 if success else 1
99+
except Exception as e:
100+
print("Error during HTTP test:", file=sys.stderr)
101+
print(str(e), file=sys.stderr)
102+
traceback.print_exc(file=sys.stderr)
103+
return 1
97104

98105
if __name__ == "__main__":
99106
sys.exit(main())

‎mcp_testing/stdio/tester.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ def __init__(self, server_command: str, args: List[str] = None, debug: bool = Fa
3939
if debug:
4040
logger.setLevel(logging.DEBUG)
4141

42-
logger.debug(f"Initialized tester with command: {server_command} {' '.join(args)}")
42+
# Only log if debug is enabled and using the instance args (which is guaranteed to be a list)
43+
logger.debug(f"Initialized tester with command: {server_command} {' '.join(self.args)}")
4344

4445
def start_server(self) -> bool:
4546
"""Start the server process.

‎tests/unit/http/test_cli.py

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) 2025 Scott Wilcox
3+
# SPDX-License-Identifier: AGPL-3.0-or-later
4+
5+
"""
6+
Unit tests for the HTTP CLI module
7+
"""
8+
9+
import unittest
10+
from unittest.mock import patch, MagicMock
11+
import sys
12+
from io import StringIO
13+
14+
from mcp_testing.http.cli import main, run_http_tester
15+
16+
17+
class TestHttpCli(unittest.TestCase):
18+
"""Tests for the HTTP CLI module."""
19+
20+
@patch('mcp_testing.http.cli.wait_for_server')
21+
@patch('mcp_testing.http.cli.run_http_tester')
22+
def test_main_success(self, mock_run_tester, mock_wait_for_server):
23+
"""Test that main returns 0 when tests pass."""
24+
# Setup
25+
mock_wait_for_server.return_value = True
26+
mock_run_tester.return_value = True
27+
28+
# Redirect stdout for testing
29+
original_stdout = sys.stdout
30+
sys.stdout = StringIO()
31+
32+
try:
33+
# Run with test arguments
34+
with patch('sys.argv', ['cli.py', '--server-url', 'http://test-server:8000/mcp']):
35+
result = main()
36+
37+
# Verify
38+
self.assertEqual(result, 0)
39+
mock_wait_for_server.assert_called_once()
40+
mock_run_tester.assert_called_once_with('http://test-server:8000/mcp', False, '2025-03-26')
41+
self.assertIn("All HTTP tests passed", sys.stdout.getvalue())
42+
finally:
43+
# Restore stdout
44+
sys.stdout = original_stdout
45+
46+
@patch('mcp_testing.http.cli.wait_for_server')
47+
@patch('mcp_testing.http.cli.run_http_tester')
48+
def test_main_failure(self, mock_run_tester, mock_wait_for_server):
49+
"""Test that main returns 1 when tests fail."""
50+
# Setup
51+
mock_wait_for_server.return_value = True
52+
mock_run_tester.return_value = False
53+
54+
# Redirect stderr for testing
55+
original_stderr = sys.stderr
56+
sys.stderr = StringIO()
57+
58+
try:
59+
# Run with test arguments
60+
with patch('sys.argv', ['cli.py']):
61+
result = main()
62+
63+
# Verify
64+
self.assertEqual(result, 1)
65+
mock_wait_for_server.assert_called_once()
66+
mock_run_tester.assert_called_once()
67+
self.assertIn("Some HTTP tests failed", sys.stderr.getvalue())
68+
finally:
69+
# Restore stderr
70+
sys.stderr = original_stderr
71+
72+
@patch('mcp_testing.http.cli.wait_for_server')
73+
def test_main_server_unreachable(self, mock_wait_for_server):
74+
"""Test that main returns 1 when server is unreachable."""
75+
# Setup
76+
mock_wait_for_server.return_value = False
77+
78+
# Run with test arguments
79+
with patch('sys.argv', ['cli.py']):
80+
result = main()
81+
82+
# Verify
83+
self.assertEqual(result, 1)
84+
mock_wait_for_server.assert_called_once()
85+
86+
@patch('mcp_testing.http.cli.MCPHttpTester')
87+
def test_run_http_tester(self, mock_tester_class):
88+
"""Test that run_http_tester creates a tester and runs all tests."""
89+
# Setup the mock
90+
mock_tester = MagicMock()
91+
mock_tester_class.return_value = mock_tester
92+
mock_tester.run_all_tests.return_value = True
93+
94+
# Call the function
95+
result = run_http_tester("http://test-server:8000/mcp", True, "2024-11-05")
96+
97+
# Verify
98+
self.assertTrue(result)
99+
mock_tester_class.assert_called_once_with("http://test-server:8000/mcp", True)
100+
self.assertEqual(mock_tester.protocol_version, "2024-11-05")
101+
mock_tester.run_all_tests.assert_called_once()
102+
103+
104+
if __name__ == '__main__':
105+
unittest.main()

0 commit comments

Comments
 (0)