Skip to content

Commit

Permalink
Allow multiple input test cases (#85)
Browse files Browse the repository at this point in the history
  • Loading branch information
@coilysiren
coilysiren authored Oct 16, 2023
1 parent c12f106 commit 2669a49
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 23 deletions.
19 changes: 14 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,20 @@ You will get output like so:
```bash
$ invoke test python any

> 🟢 script "./src/python/sort_builtin.py" succeeded
> 🟢 script "./src/python/sort_bubble_sort.py" succeeded
> 🟢 script "./src/python/sort_selection_sort.py" succeeded
> 🟢 script "./src/python/sort_insertion_sort.py" succeeded
> ✨ script run success ✨
docker run ...
⏱ ./src/python/sort_builtin.py on ./data/sort_input_0.txt ran for 0.5 seconds
🟢 ./src/python/sort_builtin.py on ./data/sort_input_0.txt succeeded
docker run ...
⏱ ./src/python/sort_bubble_sort.py on ./data/sort_input_0.txt ran for 0.51 seconds
🟢 ./src/python/sort_bubble_sort.py on ./data/sort_input_0.txt succeeded
docker run ...
⏱ ./src/python/sort_selection_sort.py on ./data/sort_input_0.txt ran for 0.58 seconds
🟢 ./src/python/sort_selection_sort.py on ./data/sort_input_0.txt succeeded
docker run ...
⏱ ./src/python/sort_insertion_sort.py on ./data/sort_input_0.txt ran for 0.52 seconds
🟢 ./src/python/sort_insertion_sort.py on ./data/sort_input_0.txt succeeded

✨ script run success ✨
```

_(note: I will likely not bother to update the above example when the output changes in minor ways)_
Expand Down
0 data/sort_input.txt → data/sort_input_0.txt
View file
File renamed without changes.
0 data/sort_output.txt → data/sort_output_0.txt
View file
File renamed without changes.
3 changes: 0 additions & 3 deletions src/readme.md
View file

This file was deleted.

54 changes: 39 additions & 15 deletions tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,12 @@ class TestRunnerContext:
script_name: str
docker_pull: str
docker_run_test: str
script_type: str
script_relative_path: str
script_name_with_file_type: str
script_output_file_path: str
script_output_file_name: str
input_file_path: str
prepared_file_path: str
snippet_start_line: int
snippet_end_line: int
Expand Down Expand Up @@ -73,23 +75,31 @@ def __init__(self, language) -> None:
# generate the contexts
self.ctxs = []
for script_path in glob.glob(f"{self.base_directory}/src/{language}/*"):
# given "src/python/sort_builtin.py" => return "sort"
script_type = script_path.split("/")[-1].split("_")[0]

# ignore helpers, metadata files, etc
if config.get("ignoreFiles") and script_path.split("/")[-1] in config.get(
"ignoreFiles"
):
continue

# ignore directories, generally compiled code
if not os.path.isfile(script_path):
continue
# generate a context for this particular script
if ctx := self.generate(language, config, script_path):
self.ctxs.append(ctx)

for input_file_path in glob.glob(
f"{self.data_folder_path}/{script_type}_input_*.txt"
):
# generate a context for this particular script
if ctx := self.generate(language, config, script_path, input_file_path):
self.ctxs.append(ctx)

@property
def data(self):
return [ctx.data for ctx in self.ctxs]

def generate(self, language, config, script_path):
def generate(self, language, config, script_path, input_file_path):
# given "src/python/sort_builtin.py" => split on "/" and return "sort_builtin.py"
script_path_split_on_slash = script_path.split("/")
script_name_with_file_type = script_path_split_on_slash[-1]
Expand All @@ -106,11 +116,14 @@ def generate(self, language, config, script_path):

# get the path of the file that's been prepared in advance
# and has the output we would be expecting from out script
prepared_file_path = f"{self.data_folder_path}/{script_type}_output.txt"
prepared_file_path = input_file_path.replace("input", "output")

# given "data/sort_input_1.txt" => return "1"
prepared_file_index = prepared_file_path.split("_")[-1].split(".")[0]

# our scripts write their output files to this path
script_output_file_name = (
f"output_{script_type}_via_{language}_{script_name}.txt"
f"output_{language}_{script_name}_{prepared_file_index}.txt"
)
script_output_file_path = f"{self.data_folder_path}/{script_output_file_name}"

Expand Down Expand Up @@ -142,7 +155,7 @@ def generate(self, language, config, script_path):

# construct ending call args
docker_run_test_list += [
f"-e=INPUT_PATH={self.data_folder_path}/{script_type}_input.txt",
f"-e=INPUT_PATH={input_file_path}",
f"-e=OUTPUT_PATH={script_output_file_path}",
config["dockerImage"],
*script_invoker,
Expand Down Expand Up @@ -170,14 +183,14 @@ def generate(self, language, config, script_path):
if snippet_start_line != 0:
raise Exception(
f'Found multiple "{self.snippet_start_text}" lines in {script_relative_path}.\n'
f"The text with found on lines {snippet_start_line - snippet_start_line_offset + 1} and {idx + 1}."
f"The lines were {snippet_start_line - snippet_start_line_offset + 1} and {idx + 1}."
)
snippet_start_line = idx + 3
if self.snippet_end_text in line:
if snippet_end_line != 0:
raise Exception(
f'Found multiple "{self.snippet_end_text}" lines in {script_relative_path}.\n'
f"The text with found on lines {snippet_end_line + snippet_end_line_offset + 1} and {idx + 1}."
f"The lines were {snippet_end_line + snippet_end_line_offset + 1} and {idx + 1}."
)
snippet_end_line = idx - snippet_end_line_offset
if snippet_start_line == 0:
Expand All @@ -195,10 +208,12 @@ def generate(self, language, config, script_path):
script_name=script_name,
docker_pull=docker_pull,
docker_run_test=docker_run_test,
script_type=script_type,
script_relative_path=script_relative_path,
script_name_with_file_type=script_name_with_file_type,
script_output_file_path=script_output_file_path,
script_output_file_name=script_output_file_name,
input_file_path=input_file_path,
prepared_file_path=prepared_file_path,
snippet_start_line=snippet_start_line,
snippet_end_line=snippet_end_line,
Expand Down Expand Up @@ -244,19 +259,24 @@ def run_tests(self, input_script):
# report timing
# we round the number so humans dont over-index on small differences
print(
f'\t⏱ script "{ctx.script_relative_path}" run for {round(end_time - start_time, 2)} seconds'
f"\t⏱ {ctx.script_relative_path} on {ctx.input_file_path} "
f"ran for {round(end_time - start_time, 2)} seconds"
)

# check if the script invoke failed
if output.exited != 0:
self.set_success_status(False)
print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
print(
f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
)
print(f'\t\t the exit code "{output.exited}" was not 0')

# check if the output file was created
if not os.path.exists(ctx.script_output_file_path):
self.set_success_status(False)
print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
print(
f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
)
print(
f"\t\t the output {ctx.script_output_file_name} file was not created"
)
Expand All @@ -266,14 +286,18 @@ def run_tests(self, input_script):
ctx.prepared_file_path, ctx.script_output_file_path
):
self.set_success_status(True)
print(f'\t🟢 script "{ctx.script_relative_path}" succeeded')
print(
f"\t🟢 {ctx.script_relative_path} on {ctx.input_file_path} succeeded"
)

# check if the output file does not match the prepared file
if os.path.exists(ctx.script_output_file_path) and not filecmp.cmp(
ctx.prepared_file_path, ctx.script_output_file_path
):
self.set_success_status(False)
print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
print(
f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
)
print(
f"\t\t output file {ctx.script_output_file_name} has does not match the prepared file"
)
Expand Down Expand Up @@ -370,4 +394,4 @@ def test(ctx: invoke.Context, language, input_script):

@invoke.task
def clean(ctx: invoke.Context):
ctx.run("git clean -fdx ./data/*")
ctx.run("git clean -fdx ./data/output_*")

0 comments on commit 2669a49

Please sign in to comment.