Skip to content

Add unit tests to rvdss #1561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 25 commits into from
Jul 18, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
49e67a9
test outline
nmdefries Nov 22, 2024
b0fa747
add regex as dependency
nmdefries Nov 22, 2024
e6d9053
add unidecode as dependency
nmdefries Nov 22, 2024
c7a4203
import relative to delphi.epidata
nmdefries Nov 22, 2024
8c6b555
switch all rvdss tests from unittest to pytest; basic abbr virus tests
nmdefries Nov 22, 2024
c0742fc
Update test_utils.py
cchuong Jan 24, 2025
b2e5013
Add tests and testdata
cchuong Mar 7, 2025
6353b18
Add utils tests and move test data
cchuong Apr 2, 2025
e503e97
Add testdata of historic reports
cchuong Apr 4, 2025
5662a79
Add additional tests and testdata
cchuong Apr 24, 2025
22c8589
Update sql table definitions and add extra na values to historic data…
cchuong Apr 25, 2025
9d22911
Add extra values that should be read as NA and counts with spaces in …
cchuong Apr 25, 2025
901b2fe
update sql keys
cchuong Apr 25, 2025
1214e70
Update pull_historic.py
cchuong Apr 25, 2025
4e5528b
Update rvdss.sql
cchuong May 8, 2025
908df83
remove unused code and table definition
cchuong May 13, 2025
02de2eb
add extra edge cases, and duplicate checking
cchuong May 15, 2025
dc14c71
remove saving to csv
cchuong May 15, 2025
4d283a6
remove scripts for manual testing
cchuong May 15, 2025
58a1478
add extra duplication checks and check if tables exists
cchuong May 16, 2025
f96d926
add logger
cchuong May 25, 2025
7cf0116
add basic integration tests
cchuong May 30, 2025
38bdd50
Combine multiple tables into one
cchuong Jul 7, 2025
5ba4fd5
stop scraping unused table
cchuong Jul 8, 2025
277c44b
skeleton integration tests
cchuong Jul 18, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions integrations/acquisition/rvdss/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import sys
import os

sys.path.append(os.getcwd())
99 changes: 99 additions & 0 deletions integrations/acquisition/rvdss/test_scenarios.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
"""Integration tests for acquisition of rvdss data."""
# standard library
import unittest
from unittest.mock import MagicMock

# first party
from delphi.epidata.client.delphi_epidata import Epidata
from delphi.epidata.acquisition.rvdss.database import update
import delphi.operations.secrets as secrets

# third party
import mysql.connector

# py3tester coverage target (equivalent to `import *`)
# __test_target__ = 'delphi.epidata.acquisition.covid_hosp.facility.update'

NEWLINE="\n"

class AcquisitionTests(unittest.TestCase):

def setUp(self):
"""Perform per-test setup."""

# configure test data
# self.test_utils = UnitTestUtils(__file__)

# use the local instance of the Epidata API
Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
Epidata.auth = ('epidata', 'key')

# use the local instance of the epidata database
secrets.db.host = 'delphi_database_epidata'
secrets.db.epi = ('user', 'pass')

# clear relevant tables
u, p = secrets.db.epi
cnx = mysql.connector.connect(user=u, password=p, database="epidata")
cur = cnx.cursor()

cur.execute('truncate table rvdss_repiratory_detections')
cur.execute('truncate table rvdss_pct_positive')
cur.execute('truncate table rvdss_detections_counts')
cur.execute('delete from api_user')
cur.execute('insert into api_user(api_key, email) values ("key", "emai")')

def test_rvdss_repiratory_detections(self):
# make sure the data does not yet exist
with self.subTest(name='no data yet'):
response = Epidata.rvdss_repiratory_detections(
'450822', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], -2, response)

# acquire sample data into local database
with self.subTest(name='first acquisition'):
acquired = Update.run(network=mock_network)
self.assertTrue(acquired)

# make sure the data now exists
with self.subTest(name='initial data checks'):
expected_spotchecks = {
"hospital_pk": "450822",
"collection_week": 20201030,
"publication_date": 20210315,
"previous_day_total_ed_visits_7_day_sum": 536,
"total_personnel_covid_vaccinated_doses_all_7_day_sum": 18,
"total_beds_7_day_avg": 69.3,
"previous_day_admission_influenza_confirmed_7_day_sum": -999999
}
response = Epidata.covid_hosp_facility(
'450822', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 2)
row = response['epidata'][0]
for k,v in expected_spotchecks.items():
self.assertTrue(
k in row,
f"no '{k}' in row:\n{NEWLINE.join(sorted(row.keys()))}"
)
if isinstance(v, float):
self.assertAlmostEqual(row[k], v, f"row[{k}] is {row[k]} not {v}")
else:
self.assertEqual(row[k], v, f"row[{k}] is {row[k]} not {v}")

# expect 113 fields per row (114 database columns, except `id`)
self.assertEqual(len(row), 113)

# re-acquisition of the same dataset should be a no-op
with self.subTest(name='second acquisition'):
acquired = Update.run(network=mock_network)
self.assertFalse(acquired)

# make sure the data still exists
with self.subTest(name='final data checks'):
response = Epidata.covid_hosp_facility(
'450822', Epidata.range(20200101, 20210101))
self.assertEqual(response['result'], 1)
self.assertEqual(len(response['epidata']), 2)


152 changes: 152 additions & 0 deletions integrations/server/test_rvdss.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
# first party
from delphi.epidata.common.integration_test_base_class import DelphiTestBase


class rvdssTest(DelphiTestBase):
"""Basic integration tests for rvdss endpoint."""

def localSetUp(self):
self.truncate_tables_list = ["rvdss_repiratory_detections",
"rvdss_pct_positive",
"rvdss_detections_counts"]

def test_rvdss_repiratory_detections(self):
"""Basic integration test for rvdss_repiratory_detections endpoint"""
self.cur.execute(
"INSERT INTO `rvdss_repiratory_detections`(`epiweek`, `time_value`, `issue`, `geo_type`, `geo_value`, `sarscov2_tests`, `sarscov2_positive_tests`, `flu_tests`, `flu_positive_tests`, `fluah1n1pdm09_positive_tests`, `fluah3_positive_tests`, `fluauns_positive_tests`, `flua_positive_tests`, `flub_positive_tests`, `rsv_tests`, `rsv_positive_tests`, `hpiv_tests`, `hpiv1_positive_tests`, `hpiv2_positive_tests`, `hpiv3_positive_tests`, `hpiv4_positive_tests`, `hpivother_positive_tests`, `adv_tests`, `adv_positive_tests`, `hmpv_tests`, `hmpv_positive_tests`, `evrv_tests`, `evrv_positive_tests`, `hcov_tests`, `hcov_positive_tests`, `week`, `weekorder`, `year`) VALUES(%(epiweek)s, %(time_value)s, %(issue)s, %(geo_type)s, %(geo_value)s, %(sarscov2_tests)s, %(sarscov2_positive_tests)s, %(flu_tests)s, %(flu_positive_tests)s, %(fluah1n1pdm09_positive_tests)s, %(fluah3_positive_tests)s, %(fluauns_positive_tests)s, %(flua_positive_tests)s, %(flub_positive_tests)s, %(rsv_tests)s, %(rsv_positive_tests)s, %(hpiv_tests)s, %(hpiv1_positive_tests)s, %(hpiv2_positive_tests)s, %(hpiv3_positive_tests)s, %(hpiv4_positive_tests)s, %(hpivother_positive_tests)s, %(adv_tests)s, %(adv_positive_tests)s, %(hmpv_tests)s, %(hmpv_positive_tests)s, %(evrv_tests)s, %(evrv_positive_tests)s, %(hcov_tests)s, %(hcov_positive_tests)s, %(week)s, %(weekorder)s, %(year)s)",
("201212", "2012-04-12", "2012-04-17", "region","on", "10", "1", "9", "1", "0", "0", "2", "3", "1", "8", "2", "7", "1", "1", "1", "1","1", "6", "5", "100", "13", "92", "9", "167", "52", "12", "34", "2012"),
)
self.cnx.commit()
response = self.epidata_client.rvdss_repiratory_detections(epiweeks=201212, geo_value="on")
self.assertEqual(
response,
{
"epidata": [
{'epiweek':201212,
'time_value':"2012-04-12",
'issue':"2012-04-17",
'geo_type':"region",
'geo_value':"on",
'sarscov2_tests':10,
'sarscov2_positive_tests':1,
'flu_tests':9,
'flu_positive_tests':1,
'fluah1n1pdm09_positive_tests':0,
'fluah3_positive_tests':0,
'fluauns_positive_tests':2,
'flua_positive_tests':3,
'flub_positive_tests':1,
'rsv_tests':8,
'rsv_positive_tests':2,
'hpiv_tests':8,
'hpiv1_positive_tests':1,
'hpiv2_positive_tests':1,
'hpiv3_positive_tests':1,
'hpiv4_positive_tests':1,
'hpivother_positive_tests':1,
'adv_tests':6,
'adv_positive_tests':5,
'hmpv_tests':100,
'hmpv_positive_tests':13,
'evrv_tests':92,
'evrv_positive_tests':9,
'hcov_tests':167,
'hcov_positive_tests':52,
'week':12,
'weekorder':34,
'year':2012
}
],
"result": 1,
"message": "success",
},
)

def test_rvdss_pct_positive(self):
"""Basic integration test for rvdss_pct_positive endpoint"""
self.cur.execute(
"INSERT INTO rvdss_pct_positive (`epiweek`, `time_value`, `issue`, `geo_type`, `geo_value`, `evrv_pct_positive`, `evrv_tests`, `evrv_positive_tests`, `hpiv_pct_positive`, `hpiv_tests`, `hpiv_positive_tests`, `adv_pct_positive`, `adv_tests`, `adv_positive_tests`,`hcov_pct_positive`, `hcov_tests`, `hcov_positive_tests`, `flua_pct_positive`, `flub_pct_positive`, `flu_tests`, `flua_positive_tests`, `flua_tests`, `flub_tests`, `flub_positive_tests`, `flu_positive_tests`, `flu_pct_positive`, `hmpv_pct_positive`, `hmpv_tests`, `hmpv_positive_tests`, `rsv_pct_positive`, `rsv_tests`, `rsv_positive_tests`, `sarscov2_pct_positive`, `sarscov2_tests`, `sarscov2_positive_tests`, `region`, `week`, `weekorder`, `year`) VALUES (%(epiweek)s, %(time_value)s, %(issue)s, %(geo_type)s, %(geo_value)s, %(evrv_pct_positive)s, %(evrv_tests)s, %(evrv_positive_tests)s, %(hpiv_pct_positive)s, %(hpiv_tests)s, %(hpiv_positive_tests)s, %(adv_pct_positive)s, %(adv_tests)s, %(hcov_pct_positive)s, %(hcov_tests)s, %(hcov_positive_tests)s, %(flua_pct_positive)s, %(flub_pct_positive)s, %(flu_tests)s, %(flua_positive_tests)s, %(flua_tests)s, %(flub_tests)s, %(flub_positive_tests)s, %(flu_positive_tests)s, %(flu_pct_positive)s, %(hmpv_pct_positive)s, %(hmpv_tests)s, %(hmpv_positive_tests)s, %(rsv_pct_positive)s, %(rsv_tests)s, %(rsv_positive_tests)s, %(sarscov2_pct_positive)s, %(sarscov2_tests)s, %(sarscov2_positive_tests)s, %(region)s, %(week)s, %(weekorder)s, %(year)s)",
("201212", "2012-04-12", "2012-04-17", "region","on","0.1","10","1","0.1","10","1","0.1","10","1","0.1","10","1","0.05","0.05","100", "10", "100","100", "10", "20", "0.1","0.1","10","1","0.1","10","1","0.1","10","1","on","12","34","2012")
)
self.cnx.commit()
response = self.epidata_client.rvdss_pct_positive(epiweeks=201212, geo_value="on")
self.assertEqual(
response,
{
"epidata": [
{'epiweek':201212,
'time_value':"2012-04-12",
'issue':"2012-04-17",
'geo_type':"region",
'geo_value':"on",
'evrv_pct_positive':0.1,
'evrv_tests':10,
'evrv_positive_tests':1,
'hpiv_pct_positive':0.1,
'hpiv_tests':10,
'hpiv_positive_tests':1,
'adv_pct_positive':0.1,
'adv_tests':10,
'adv_positive_tests':1,
'hcov_pct_positive':0.1,
'hcov_tests':10,
'hcov_positive_tests':1,
'flua_pct_positive':0.05,
'flub_pct_positive':0.05,
'flu_tests':100,
'flua_positive_tests':10,
'flua_tests':100,
'flub_tests':100,
'flub_positive_tests':10,
'flu_positive_tests':20,
'flu_pct_positive':0.1,
'hmpv_pct_positive':0.1,
'hmpv_tests':10,
'hmpv_positive_tests':1,
'rsv_pct_positive':0.1,
'rsv_tests':10,
'rsv_positive_tests':1,
'sarscov2_pct_positive':0.1,
'sarscov2_tests':10,
'sarscov2_positive_tests':1,
'region':"on",
'week':12,
'weekorder':34,
'year':2012
}
],
"result": 1,
"message": "success",
},
)

def test_rvdss_detections_counts(self):
"""Basic integration test for rvdss_detections_counts endpoint"""
self.cur.execute(
"INSERT INTO rvdss_detections_counts (`epiweek`, `time_value`, `issue`, `geo_type`, `geo_value`, `hpiv_positive_tests`, `adv_positive_tests`, `hmpv_positive_tests`, `evrv_positive_tests`, `hcov_positive_tests`, `rsv_positive_tests`, `flu_positive_tests`) VALUES (%(epiweek)s, %(time_value)s, %(issue)s, %(geo_type)s, %(geo_value)s, %(hpiv_positive_tests)s, %(adv_positive_tests)s, %(hmpv_positive_tests)s, %(evrv_positive_tests)s, %(hcov_positive_tests)s, %(rsv_positive_tests)s, %(flu_positive_tests)s)",
("201212", "2012-04-12", "2012-04-17", "nation","ca", "10", "9", "8", "7", "6", "5", "4"),
)
self.cnx.commit()
response = self.epidata_client.rvdss_detections_counts(epiweeks=201212, geo_value="ca")
self.assertEqual(
response,
{
"epidata": [
{'epiweek':201212,
'time_value':"2012-04-12",
'issue':"2012-04-17",
'geo_type':"nation",
'geo_value':"ca",
'hpiv_positive_tests':10,
'adv_positive_tests':9,
'hmpv_positive_tests':8,
'evrv_positive_tests':7,
'hcov_positive_tests':6,
'rsv_positive_tests':5,
'flu_positive_tests':4
}
],
"result": 1,
"message": "success",
},
)
2 changes: 2 additions & 0 deletions requirements.api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ pandas==1.2.3
python-dotenv==0.15.0
pyyaml
redis==3.5.3
regex
requests==2.32.0
scipy==1.10.0
sentry-sdk[flask]
SQLAlchemy==1.4.40
structlog==22.1.0
tenacity==7.0.0
typing-extensions
unidecode
werkzeug==2.3.8
3 changes: 3 additions & 0 deletions requirements.dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,6 @@ selenium==4.7.2
sqlalchemy-stubs>=0.3
tenacity==7.0.0
xlrd==2.0.1
bs4
mock
requests_file
4 changes: 4 additions & 0 deletions src/acquisition/rvdss/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@
'prairies', 'pr', "british columbia",'bc',"territories",'terr',]
NATION = ["canada","can",'ca',]

PROVINCES = ['nl','pe','ns','nb','qc','on','mb','sk','ab','bc','yt','nt','nu']

# Construct dashboard and data report URLS.
DASHBOARD_BASE_URL = "https://health-infobase.canada.ca/src/data/respiratory-virus-detections/"
DASHBOARD_W_DATE_URL = DASHBOARD_BASE_URL + "archive/{date}/"
Expand Down Expand Up @@ -115,3 +117,5 @@

UPDATE_DATES_FILE = "update_dates.txt"
NOW = datetime.now()

LOGGER_FILENAME = "rvdss.log"
Loading
Loading