em2ex/run_tests.py at master · cpgr/em2ex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#!/usr/bin/env python

import subprocess
import pytest
import os
import yaml

# Exception class to throw exception for pytest
class Em2exException(Exception):
    pass

# Run the current file using pytest
pytest.main(['-v', '-rsx', '--tb=line', 'run_tests.py'])

# Find all 'tests' yml files that contain the test specifications
tests_files = []
for root, dirs, files in os.walk("test/", topdown=False):
    for name in files:
        if name == 'tests':
            tests_files.append(os.path.join(root, name))

# Read yaml data from tests files
tests = {}
for test in tests_files:
    with open(test, 'r') as file:
        testcfg = yaml.safe_load(file)

        # Extract the filepath
        filepath, filename = os.path.split(test)

        for key, values in testcfg.items():
            # Make keys unique by prepending the filepath
            fullpathkey = os.path.join(filepath, key)
            tests[fullpathkey] = values
            tests[fullpathkey]['filepath'] = filepath

# Run all em2ex tests found during search
@pytest.mark.parametrize('key', tests)
def test_em2ex(key, use_official_api, exodiff):
    ''' Run all em2ex tests using appropriate test function '''

    # If the type key isn't specified, skip test
    if 'type' not in tests[key].keys():
        pytest.skip(key + ': Skipped as test type not specified')

    # If the filename key isn't specified, skip test
    if 'filename' not in tests[key].keys():
        pytest.skip(key + ': Skipped as file not specified')

    # If the test type is exodiff, run exodiff_test
    if tests[key]['type'] == 'exodiff':
        # If the gold key isn't specified, skip test
        if 'gold' not in tests[key].keys():
            pytest.skip(key + ': Skipped as gold file not specified')
        else:
            exodiff_test(key, use_official_api, exodiff)

    # If the test type is exception, run the expected_error test
    elif tests[key]['type'] == 'exception':
        # If the expected_error key isn't specified, skip test
        if 'expected_error' not in tests[key].keys():
            pytest.skip(key + ': Skipped as expected_error not specified')
        else:
            with pytest.raises(Exception) as excinfo:
                expected_error(key)

            assert tests[key]['expected_error'] in str(excinfo.value)

    # If the test type is output, check that expected text appears in stdout
    elif tests[key]['type'] == 'output':
        if 'expected_output' not in tests[key].keys():
            pytest.skip(key + ': Skipped as expected_output not specified')
        else:
            check_output(key)

    else:
        # Skip unknown test type
        pytest.skip(key + ': Skipped as unknown test type')

    return

def exodiff_test(key, use_official_api, exodiff):
    ''' Convert reservoir model to exodus and compare with gold file '''

    # Convert reservoir model to Exodus II model
    filepath = tests[key]['filepath']

    filename = tests[key]['filename']
    testfilename = os.path.join(filepath, filename)

    command = ['./em2ex.py']
    arguments = ['-f']

    if use_official_api:
        arguments.append('--use-official-api')

    if 'cli_args' in tests[key].keys():
        arguments.extend(tests[key]['cli_args'].split())

    command.extend(arguments)
    command.append(testfilename)
    subprocess.check_output(command)

    # Compare the converted model with a gold file using exodiff
    try:
        filename_base, file_extension = os.path.splitext(filename)
        exodus_filename = os.path.join(filepath, filename_base + '.e')
        gold_filename = os.path.join(filepath, 'gold', tests[key]['gold'])
        subprocess.check_output([exodiff, '--quiet', exodus_filename, gold_filename])

    except subprocess.CalledProcessError:
        raise Em2exException(key + ': exodiff failed - files are different')

    return

def expected_error(key):
    ''' Raise an exception when an error is thrown while em2ex is running '''

    # Convert reservoir model to Exodus II model
    filepath = tests[key]['filepath']

    filename = tests[key]['filename']
    testfilename = os.path.join(filepath, filename)

    command = ['./em2ex.py', '-f']
    if 'cli_args' in tests[key].keys():
        command.extend(tests[key]['cli_args'].split())
    command.append(testfilename)

    result = subprocess.run(command, capture_output=True, text=True)

    raise Em2exException(result.stdout + result.stderr)

    return

def check_output(key):
    ''' Check that expected text appears in em2ex stdout on a successful run '''

    filepath = tests[key]['filepath']
    filename = tests[key]['filename']
    testfilename = os.path.join(filepath, filename)

    command = ['./em2ex.py', '-f']
    if 'cli_args' in tests[key].keys():
        command.extend(tests[key]['cli_args'].split())
    command.append(testfilename)

    result = subprocess.run(command, capture_output=True, text=True)
    combined = result.stdout + result.stderr

    assert tests[key]['expected_output'] in combined, \
        '{}: expected output not found.\nExpected: {}\nGot: {}'.format(
            key, tests[key]['expected_output'], combined)

    return