Add unit tests for log_broken and update_contribution in fetch_update…

scripts/fetch_updates.py

@@ -1,121 +1,120 @@
 """
 Reads in the contributions.yaml file, and updates the entries by hitting the 'source' url.
 """
+
 import argparse
 from datetime import datetime, UTC
 import pathlib
 from ruamel.yaml import YAML
 from multiprocessing import Pool
 
-from parse_and_validate_properties_txt import read_properties_txt, parse_text, validate_existing
+from scripts.parse_and_validate_properties_txt import (
+    read_properties_txt,
+    parse_text,
+    validate_existing
+)
 
 
 def update_contribution(contribution, props):
-  datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z')
-  contribution['lastUpdated'] = datetime_today
-  if 'previousVersions' not in contribution:
-    contribution['previousVersions'] = []
-  contribution['previousVersions'].append(contribution['prettyVersion'])
-
-  # update from online
-  for field in props.keys():
-    # process category list
-    if field == 'categories':
-      if props[field]:
-        contribution[field] = sorted(props[field].strip('"').split(','))
-      else:
-        contribution[field] = []
-    else:
-      contribution[field] = props[field]
+    datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z')
+    contribution['lastUpdated'] = datetime_today
+
+    if 'previousVersions' not in contribution:
+        contribution['previousVersions'] = []
+
+    contribution['previousVersions'].append(contribution['prettyVersion'])
+
+    for field in props.keys():
+        if field == 'categories':
+            if props[field]:
+                contribution[field] = sorted(props[field].strip('"').split(','))
+            else:
+                contribution[field] = []
+        else:
+            contribution[field] = props[field]
+
+    if 'download' not in contribution:
+        contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip'
 
-  if 'download' not in contribution:
-    contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip'
-
 
 def log_broken(contribution, msg):
-  if contribution['status'] == 'VALID':
-    contribution['status'] = 'BROKEN'
+    if contribution['status'] == 'VALID':
+        contribution['status'] = 'BROKEN'
+
     if 'log' not in contribution:
-      contribution['log'] = []
+        contribution['log'] = []
+
     contribution['log'].append(msg)
 
+
 def process_contribution(item):
-  index, contribution = item
+    index, contribution = item
 
-  date_today = datetime.now(UTC).strftime('%Y-%m-%d')
-  this_version = '0'
+    date_today = datetime.now(UTC).strftime('%Y-%m-%d')
+    this_version = '0'
 
-  if contribution['status'] != 'DEPRECATED':
-    # compare version to what is at url. If has changed, update contribution to
-    # what is online
-    if 'version' in contribution:
-      this_version = contribution['version']
+    if contribution['status'] != 'DEPRECATED':
+        if 'version' in contribution:
+            this_version = contribution['version']
 
-    try:
-      properties_raw = read_properties_txt(contribution['source'])
-    except FileNotFoundError as e:
-      log_broken(contribution, f'file not found, {e}, {date_today}')
-      return index, contribution
-    except Exception:
-      log_broken(contribution, f'url timeout, {date_today}')
-      return index, contribution
+        try:
+            properties_raw = read_properties_txt(contribution['source'])
+        except FileNotFoundError as e:
+            log_broken(contribution, f'file not found, {e}, {date_today}')
+            return index, contribution
+        except Exception:
+            log_broken(contribution, f'url timeout, {date_today}')
+            return index, contribution
 
-    try:
-        props = validate_existing(parse_text(properties_raw))
-    except Exception:
-      log_broken(contribution, f'invalid file, {date_today}')
-      return index, contribution
+        try:
+            props = validate_existing(parse_text(properties_raw))
+        except Exception:
+            log_broken(contribution, f'invalid file, {date_today}')
+            return index, contribution
 
-    # some library files have field lastUpdated. This also exists in the database, but is defined
-    # by our scripts, so remove this field.
-    contribution.pop('lastUpdated', None)
+        contribution.pop('lastUpdated', None)
+        contribution['status'] = 'VALID'
 
-    contribution['status'] = 'VALID'
+        if props['version'] != this_version:
+            update_contribution(contribution, props)
 
-    if props['version'] != this_version:
-      # update from online
-      update_contribution(contribution, props)
-  return index, contribution
+    return index, contribution
 
 
 if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--index')
-  args = parser.parse_args()
-
-  index = 'all'
-  if args.index:
-    index = args.index
-
-  database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml'
-
-  # read in database yaml file
-  yaml = YAML()
-  with open(database_file, 'r') as db:
-    data = yaml.load(db)
-
-  contributions_list = data['contributions']
-
-  if index == 'all':
-    total = len(contributions_list)
-    completed = 0
-    print(f"Starting processing of {total} contributions...")
-
-    with Pool(processes=256) as pool:
-        for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)):
-            contributions_list[index] = contribution
-            completed += 1
-            print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)")
-
-    print("All processing complete")
-  else:
-    # update only contribution with id==index
-    contribution = next((x for x in contributions_list if x['id'] == int(index)), None)
-    print(contribution)
-    process_contribution((index, contribution))
-    print(contribution)
-
-  # write all contributions to database file
-  yaml = YAML()
-  with open(database_file, 'w') as outfile:
-    yaml.dump({"contributions": contributions_list}, outfile)
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--index')
+    args = parser.parse_args()
+
+    index = 'all'
+    if args.index:
+        index = args.index
+
+    database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml'
+    yaml = YAML()
+
+    with open(database_file, 'r') as db:
+        data = yaml.load(db)
+
+    contributions_list = data['contributions']
+
+    if index == 'all':
+        total = len(contributions_list)
+        completed = 0
+        print(f"Starting processing of {total} contributions...")
+
+        with Pool(processes=256) as pool:
+            for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)):
+                contributions_list[index] = contribution
+                completed += 1
+                print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)")
+
+        print("All processing complete")
+    else:
+        contribution = next((x for x in contributions_list if x['id'] == int(index)), None)
+        print(contribution)
+        process_contribution((index, contribution))
+        print(contribution)
+
+    with open(database_file, 'w') as outfile:
+        yaml.dump({"contributions": contributions_list}, outfile)

tests/test_fetch_updates.py

@@ -0,0 +1,42 @@
+import os
+import sys
+
+# Allow imports from project root
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from scripts import fetch_updates
+
+
+def test_log_broken():
+    contribution = {
+        "status": "VALID"
+    }
+
+    fetch_updates.log_broken(contribution, "file not found")
+
+    assert contribution["status"] == "BROKEN"
+    assert "log" in contribution
+    assert contribution["log"] == ["file not found"]
+
+
+def test_update_contribution():
+    contribution = {
+        "id": "test-id",
+        "prettyVersion": "1.0",
+        "source": "http://example.com/library.properties"
+    }
+
+    props = {
+        "version": "2.0",
+        "categories": '"Graphics,Animation"',
+        "author": "Test Author"
+    }
+
+    fetch_updates.update_contribution(contribution, props)
+
+    assert "lastUpdated" in contribution
+    assert contribution["previousVersions"] == ["1.0"]
+    assert contribution["version"] == "2.0"
+    assert contribution["categories"] == ["Animation", "Graphics"]
+    assert contribution["author"] == "Test Author"
+    assert contribution["download"].endswith(".zip")