Argoutils cleanup

.github/workflows/ci.yml

@@ -13,9 +13,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04]
-        python-version: ['3.8']
-        toxenv: [quality, py38]
+        os: [ubuntu-latest]
+        python-version: ['3.10.11']
+        toxenv: [quality, py310]
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/publish_pypi.yml

@@ -7,14 +7,14 @@ on:
 
 jobs:
     push:
-      runs-on: ubuntu-20.04
+      runs-on: ubuntu-24.10
       steps:
         - name: Checkout
           uses: actions/checkout@v2
         - name: setup python
           uses: actions/setup-python@v2
           with:
-            python-version: 3.8
+            python-version: 3.10.11
 
         - name: Install pip
           run: pip install -r requirements/pip.txt

edx_argoutils/common.py

@@ -10,7 +10,6 @@
 from datetime import datetime, timedelta, date
 
 
-
 def get_date(date: str):
     """
     Return today's date string if date is None. Otherwise return the passed parameter value.
@@ -101,7 +100,8 @@ def get_filename_safe_course_id(course_id, replacement_char='_'):
     # TODO: Once we support courses with unicode characters, we will need to revisit this.
     return re.sub(r'[^\w\.\-]', six.text_type(replacement_char), filename)
 
-def generate_date_range(start_date= None, end_date= None, is_daily: bool = None):
+
+def generate_date_range(start_date=None, end_date=None, is_daily: bool = None):
     """
     Generate a list of dates depending on parameters passed. Dates are inclusive.
         Custom dates is top priority: start_date & end_date are set, is_daily = False
@@ -116,11 +116,11 @@ def generate_date_range(start_date= None, end_date= None, is_daily: bool = None)
 
     if start_date is not None and end_date is not None and is_daily is False:
         # Manual run: user entered parameters for custom dates
-        #logger.info("Setting dates for manual run...")
-        #start_date= start_date.strftime('%Y-%m-%d')
+        # Logger.info("Setting dates for manual run...")
+        # Start_date= start_date.strftime('%Y-%m-%d')
         start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
         end_date = datetime.strptime(end_date, "%Y-%m-%d").date()
-        #end_date= end_date.strftime('%Y-%m-%d')
+        # End_date= end_date.strftime('%Y-%m-%d')
 
     elif start_date is None and end_date is None and is_daily is True:
         # Daily run: minus 2 lag completed day, eg. if today is 9/14, output is 9/12

edx_argoutils/mysql.py

@@ -92,6 +92,9 @@ def load_s3_data_to_mysql(
                 Defaults to `False`.
       use_manifest (bool, optional): Whether to use a manifest file to load data. Defaults to `False`.
     """
+
+    if not table_columns:
+        raise ValueError("table_columns cannot be empty")
 
     def _drop_temp_tables(table, connection):
         for table in [table + '_old', table + '_temp']:

edx_argoutils/record.py

@@ -2,13 +2,19 @@
 
 import datetime
 import itertools
+import logging
 import re
 from collections import OrderedDict
 
 import ciso8601
 import pytz
 import six
-from prefect.utilities.logging import get_logger
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+)
+logger = logging.getLogger("Record Class Logger.")
 
 DEFAULT_NULL_VALUE = b'\\N'
 
@@ -443,8 +449,9 @@ def decode(self, encoded_string, _field_obj):
         if encoded_string == self.null_value:
             return None
         else:
-            return encoded_string.decode('utf8')
-
+            if isinstance(encoded_string, bytes):  # Only decode if it's bytes
+                return encoded_string.decode('utf8')
+            return encoded_string  # Already a string, return as-is
 
 class Field(object):
     """
@@ -714,6 +721,20 @@ def deserialize_from_string(self, string_value):
         # However, we assume the datetime does not include TZ info, and that it's UTC.
         return datetime.datetime(*[int(x) for x in re.split(r'\D+', string_value) if x], tzinfo=self.utc_tz)
 
+    def deserialize_from_string(self, string_value):
+        """Returns a datetime instance parsed from the numbers in the given string_value."""
+        if string_value is None:
+            return None
+        # Note: we need to be flexible here, because the datetime format differs between input sources
+        # (e.g.  tracking logs, REST API)
+        # However, we assume the datetime does not include TZ info, and that it's UTC.
+        try:
+            values = [int(x) for x in re.split(r'\D+', string_value) if x]
+            if not values:  # If the list is empty, return None instead of failing
+                return None
+            return datetime.datetime(*values, tzinfo=self.utc_tz)
+        except (ValueError, TypeError):
+            return None  # Return None for invalid inputs
 
 class FloatField(Field):  # pylint: disable=abstract-method
     """Represents a field that contains a floating point number."""
@@ -782,8 +803,6 @@ def _add_entry(self, record_dict, record_key, record_field, label, obj):
         (It's only fatal, then, if the value was required.)
         """
 
-        logger = get_logger()
-
         def backslash_encode_value(value):
             """Implement simple backslash encoding, similar to .encode('string_escape')."""
             return value.replace('\\', '\\\\').replace('\r', '\\r').replace('\t', '\\t').replace('\n', '\\n')

edx_argoutils/s3.py

@@ -7,6 +7,7 @@
 
 logger = logging.getLogger("s3")
 
+
 def get_s3_client(credentials: dict = None):
     s3_client = None
     if credentials:
@@ -21,6 +22,7 @@ def get_s3_client(credentials: dict = None):
 
     return s3_client
 
+
 def delete_s3_directory(bucket: str = None, prefix: str = None, credentials: dict = None):
     """
     Deletes all objects with the given S3 directory (prefix) from the given bucket.

edx_argoutils/sitemap.py

@@ -4,7 +4,7 @@
 
 import json
 import xml.etree.ElementTree as ET
-from os.path import basename, join, splitext
+from os.path import basename, splitext
 from urllib.parse import urlparse
 import boto3
 import requests

edx_argoutils/snowflake.py

@@ -8,7 +8,6 @@
 from typing import List, TypedDict
 from urllib.parse import urlparse
 
-import backoff
 import snowflake.connector
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization

requirements/base.in

@@ -3,8 +3,7 @@ boto3
 botocore
 ciso8601
 edx-opaque-keys
-hvac
-importlib-metadata==8.4.0  # Pinned for tox and virtualenv
-mysql-connector-python==8.0.21 # Pinned for dependency issues with prefect1.4.1
+mysql-connector-python
+importlib-metadata
 paramiko
-prefect[aws,google,snowflake,viz]==1.4.1
+snowflake-connector-python