Merge branch 'dev' into ndefries/metadata-min-issue

dev/local/setup.cfg

@@ -24,6 +24,7 @@ packages =
     delphi.epidata.acquisition.twtr
     delphi.epidata.acquisition.wiki
     delphi.epidata.client
+    delphi.epidata.common
     delphi.epidata.server
     delphi.epidata.server.admin
     delphi.epidata.server.admin.templates

docs/api/covidcast-signals/jhu-csse.md

@@ -1,6 +1,6 @@
 ---
 title: JHU Cases and Deaths
-parent: Data Sources and Signals
+parent: Inactive Signals
 grand_parent: COVIDcast Main Endpoint
 ---
 
@@ -23,12 +23,12 @@ University.
 
 | Signal and 7-day average signal | Description |
 |---|---|
-| `confirmed_cumulative_num` | Cumulative number of confirmed COVID-19 cases <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
-| `confirmed_cumulative_prop` | Cumulative number of confirmed COVID-19 cases per 100,000 population <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
+| `confirmed_cumulative_num` and `confirmed_7dav_cumulative_num` | Cumulative number of confirmed COVID-19 cases <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
+| `confirmed_cumulative_prop` and `confirmed_7dav_cumulative_prop` | Cumulative number of confirmed COVID-19 cases per 100,000 population <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
 | `confirmed_incidence_num` and `confirmed_7dav_incidence_num` | Number of new confirmed COVID-19 cases, daily <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
 | `confirmed_incidence_prop` and `confirmed_7dav_incidence_prop` | Number of new confirmed COVID-19 cases per 100,000 population, daily <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
-| `deaths_cumulative_num` | Cumulative number of confirmed deaths due to COVID-19 <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
-| `deaths_cumulative_prop` | Cumulative number of confirmed due to COVID-19, per 100,000 population <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
+| `deaths_cumulative_num` and `deaths_7dav_cumulative_num` | Cumulative number of confirmed deaths due to COVID-19 <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
+| `deaths_cumulative_prop` and `deaths_7dav_cumulative_prop` | Cumulative number of confirmed due to COVID-19, per 100,000 population <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
 | `deaths_incidence_num` and `deaths_7dav_incidence_num` | Number of new confirmed deaths due to COVID-19, daily <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
 | `deaths_incidence_prop` and `deaths_7dav_incidence_prop` | Number of new confirmed deaths due to COVID-19 per 100,000 population, daily <br/> **Earliest date available:** 2020-01-22 & 2020-02-20 |
 

docs/api/covidcast-signals/nchs-mortality.md

@@ -68,7 +68,7 @@ York State in our reports.
 
 We report the NCHS Mortality data in a weekly format (`time_type=week` &
 `time_value={YYYYWW}`, where `YYYYWW` refers to an epiweek). The CDC defines
-the [epiweek](https://wwwn.cdc.gov/nndss/document/MMWR_Week_overview.pdf) as
+the [epiweek](https://web.archive.org/web/20210623224758/https://wwwn.cdc.gov/nndss/document/MMWR_Week_overview.pdf) as
 seven days, from Sunday to Saturday. We check the week-ending dates provided in
 the NCHS morality data and use Python package
 [epiweeks](https://pypi.org/project/epiweeks/) to convert them into epiweek

docs/symptom-survey/publications.md

@@ -26,6 +26,19 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
 
 Research publications using the survey data include:
 
+- A. Srivastava, J. M. Ramirez, S. Díaz-Aranda, J. Aguilar, A. F. Anta, A. Ortega,
+  and R. E. Lillo (2024). [Nowcasting temporal trends using indirect surveys](https://doi.org/10.1609/aaai.v38i20.30242).
+  In *Proceedings of the 38th AAAI Conference on Artificial Intelligence* 38,
+  22359–22367.
+- P. Porebski, S. Venkatramanan, A. Adiga, B. Klahn, B. Hurt, M. L. Wilson,
+  J. Chen, A. Vullikanti, M. Marathe & B. Lewis (2024). [Data-driven
+  mechanistic framework with stratified immunity and effective transmissibility
+  for COVID-19 scenario projections](https://doi.org/10.1016/j.epidem.2024.100761).
+  *Epidemics* 100761.
+- V. Nelson, B. Bashyal, P.-N. Tan & Y. A. Argyris (2024). [Vaccine rhetoric
+  on social media and COVID-19 vaccine uptake rates: A triangulation using
+  self-reported vaccine acceptance](https://doi.org/10.1016/j.socscimed.2024.116775).
+  *Social Science & Medicine* 116775.
 - R.R. Andridge (2024). [Using proxy pattern-mixture models to explain bias in
   estimates of COVID-19 vaccine uptake from two large surveys](https://doi.org/10.1093/jrsssa/qnae005).
   *Journal of the Royal Statistical Society Series A: Statistics in Society*.
@@ -34,7 +47,7 @@ Research publications using the survey data include:
   *IISE Transactions*.
 - de Vries, M., Kim, J.Y. & Han, H. (2023). [The unequal landscape of civic
   opportunity in America](https://doi.org/10.1038/s41562-023-01743-1). *Nature
-  Human Behavior*. 
+  Human Behavior* 8, 256-263. 
 - E. Tuzhilina, T. J. Hastie, D. J. McDonald, J. K. Tay & R. Tibshirani (2023).
   [Smooth multi-period forecasting with application to prediction of COVID-19
   cases](https://doi.org/10.1080/10618600.2023.2285337). *Journal of Computational

integrations/client/test_delphi_epidata.py

@@ -3,6 +3,7 @@
 # standard library
 import time
 from json import JSONDecodeError
+from requests.models import Response
 from unittest.mock import MagicMock, patch
 
 # first party
@@ -41,6 +42,8 @@ def localSetUp(self):
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
+    Epidata.debug = False
+    Epidata.sandbox = False
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -221,6 +224,82 @@ def test_retry_request(self, get):
                        {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'}
                        )
 
+  @patch('requests.post')
+  @patch('requests.get')
+  def test_debug(self, get, post):
+    """Test that in debug mode request params are correctly logged."""
+    class MockResponse:
+      def __init__(self, content, status_code):
+          self.content = content
+          self.status_code = status_code
+      def raise_for_status(self): pass
+
+    Epidata.debug = True
+
+    try:
+      with self.subTest(name='test multiple GET'):
+        with self.assertLogs('delphi_epidata_client', level='INFO') as logs:
+          get.reset_mock()
+          get.return_value = MockResponse(b'{"key": "value"}', 200)
+          Epidata._request_with_retry("test_endpoint1", params={"key1": "value1"})
+          Epidata._request_with_retry("test_endpoint2", params={"key2": "value2"})
+
+        output = logs.output
+        self.assertEqual(len(output), 4) # [request, response, request, response]
+        self.assertIn("Sending GET request", output[0])
+        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint1/\"", output[0])
+        self.assertIn("\"params\": {\"key1\": \"value1\"}", output[0])
+        self.assertIn("Received response", output[1])
+        self.assertIn("\"status_code\": 200", output[1])
+        self.assertIn("\"len\": 16", output[1])
+        self.assertIn("Sending GET request", output[2])
+        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint2/\"", output[2])
+        self.assertIn("\"params\": {\"key2\": \"value2\"}", output[2])
+        self.assertIn("Received response", output[3])
+        self.assertIn("\"status_code\": 200", output[3])
+        self.assertIn("\"len\": 16", output[3])
+
+      with self.subTest(name='test GET and POST'):
+        with self.assertLogs('delphi_epidata_client', level='INFO') as logs:
+          get.reset_mock()
+          get.return_value = MockResponse(b'{"key": "value"}', 414)
+          post.reset_mock()
+          post.return_value = MockResponse(b'{"key": "value"}', 200)
+          Epidata._request_with_retry("test_endpoint3", params={"key3": "value3"})
+
+        output = logs.output
+        self.assertEqual(len(output), 3) # [request, response, request, response]
+        self.assertIn("Sending GET request", output[0])
+        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint3/\"", output[0])
+        self.assertIn("\"params\": {\"key3\": \"value3\"}", output[0])
+        self.assertIn("Received 414 response, retrying as POST request", output[1])
+        self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/test_endpoint3/\"", output[1])
+        self.assertIn("\"params\": {\"key3\": \"value3\"}", output[1])
+        self.assertIn("Received response", output[2])
+        self.assertIn("\"status_code\": 200", output[2])
+        self.assertIn("\"len\": 16", output[2])
+    finally: # make sure this global is always reset
+      Epidata.debug = False
+
+  @patch('requests.post')
+  @patch('requests.get')
+  def test_sandbox(self, get, post):
+    """Test that in debug + sandbox mode request params are correctly logged, but no queries are sent."""
+    Epidata.debug = True
+    Epidata.sandbox = True
+    try:
+      with self.assertLogs('delphi_epidata_client', level='INFO') as logs:
+        Epidata.covidcast('src', 'sig', 'day', 'county', 20200414, '01234')
+      output = logs.output
+      self.assertEqual(len(output), 1)
+      self.assertIn("Sending GET request", output[0])
+      self.assertIn("\"url\": \"http://delphi_web_epidata/epidata/covidcast/\"", output[0])
+      get.assert_not_called()
+      post.assert_not_called()
+    finally: # make sure these globals are always reset
+      Epidata.debug = False
+      Epidata.sandbox = False
+
   def test_geo_value(self):
     """test different variants of geo types: single, *, multi."""
 

requirements.api.txt

@@ -2,7 +2,7 @@ delphi_utils==0.3.15
 epiweeks==2.1.2
 Flask==2.2.5
 Flask-Limiter==3.3.0
-jinja2==3.1.3
+jinja2==3.1.4
 more_itertools==8.4.0
 mysqlclient==2.1.1
 orjson==3.9.15

requirements.dev.txt

@@ -1,4 +1,4 @@
-aiohttp==3.9.2
+aiohttp==3.9.4
 black>=20.8b1
 bump2version==1.0.1
 covidcast==0.1.5

src/client/delphi_epidata.py

@@ -10,12 +10,15 @@
 
 # External modules
 import requests
+import time
 import asyncio
 from tenacity import retry, stop_after_attempt
 
 from aiohttp import ClientSession, TCPConnector, BasicAuth
 from importlib.metadata import version, PackageNotFoundError
 
+from delphi.epidata.common.logger import get_structured_logger
+
 # Obtain package version for the user-agent. Uses the installed version by
 # preference, even if you've installed it and then use this script independently
 # by accident.
@@ -49,6 +52,10 @@ class Epidata:
 
     client_version = _version
 
+    logger = get_structured_logger('delphi_epidata_client')
+    debug = False # if True, prints extra logging statements
+    sandbox = False # if True, will not execute any queries
+
     # Helper function to cast values and/or ranges to strings
     @staticmethod
     def _listitem(value):
@@ -71,9 +78,25 @@ def _list(values):
     def _request_with_retry(endpoint, params={}):
         """Make request with a retry if an exception is thrown."""
         request_url = f"{Epidata.BASE_URL}/{endpoint}/"
+        if Epidata.debug:
+            Epidata.logger.info("Sending GET request", url=request_url, params=params, headers=_HEADERS, auth=Epidata.auth)
+        if Epidata.sandbox:
+            resp = requests.Response()
+            resp._content = b'true'
+            return resp
+        start_time = time.time()
         req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS)
         if req.status_code == 414:
+            if Epidata.debug:
+                Epidata.logger.info("Received 414 response, retrying as POST request", url=request_url, params=params, headers=_HEADERS)
             req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS)
+        if Epidata.debug:
+            Epidata.logger.info(
+                "Received response",
+                status_code=req.status_code,
+                len=len(req.content),
+                time=round(time.time() - start_time, 4)
+            )
         # handle 401 and 429
         req.raise_for_status()
         return req