Update script to pull URLs straight from Mailchimp

Author: mgax

docker-compose.yml

@@ -18,6 +18,7 @@ services:
       PGUSER: wagtailorg
       PGPASSWORD: wagtailorg
       PGDATABASE: wagtailorg
+      MAILCHIMP_API_KEY: ${MAILCHIMP_API_KEY}
     ports:
       - 8000:8000
     volumes:

wagtailio/newsletter/management/commands/fetch_newsletters.py

@@ -1,48 +1,97 @@
 from datetime import datetime
-import json
+import os
 from pathlib import Path
 
 from django.core.management.base import BaseCommand
 
+from mailchimp_marketing import Client
 import requests
 
 
-class Command(BaseCommand):
-    help = "Fetches newsletter URLs from the provided JSON file and saves them to the archive directory"
+def get_mailchimp_client():
+    """Initialize and return a Mailchimp client."""
+    api_key = os.getenv("MAILCHIMP_API_KEY")
+    if not api_key:
+        raise ValueError("MAILCHIMP_API_KEY environment variable is not set")
+
+    # Extract datacenter from API key (it's the part after the '-')
+    datacenter = api_key.split("-")[-1]
+
+    client = Client()
+    client.set_config({"api_key": api_key, "server": datacenter})
+    client.ping.get()
+    return client
+
+
+def fetch_campaign_urls():
+    """Fetch all campaign URLs from Mailchimp."""
+    client = get_mailchimp_client()
 
-    def add_arguments(self, parser):
-        parser.add_argument(
-            "input_file",
-            type=str,
-            help="Path to the JSON file containing newsletter data",
+    campaigns = []
+    offset = 0
+    count = 100  # Number of campaigns to fetch per request
+
+    while True:
+        # Get a batch of campaigns
+        response = client.campaigns.list(
+            count=count,
+            offset=offset,
+            fields=[
+                "campaigns.archive_url",
+                "campaigns.settings.title",
+                "campaigns.send_time",
+                "campaigns.id",
+                "total_items",
+            ],
         )
 
-    def handle(self, *args, **options):
-        input_file = options["input_file"]
-
-        # Read the JSON file
-        try:
-            with open(input_file) as f:
-                newsletters = json.load(f)
-        except FileNotFoundError:
-            self.stderr.write(self.style.ERROR(f"Input file not found: {input_file}"))
-            return
-        except json.JSONDecodeError:
-            self.stderr.write(self.style.ERROR(f"Invalid JSON in file: {input_file}"))
-            return
+        # Extract campaign data
+        for campaign in response["campaigns"]:
+            # Only include campaigns that have been sent
+            if campaign.get("send_time"):
+                campaigns.append(
+                    {
+                        "id": campaign["id"],
+                        "send_date": campaign["send_time"],
+                        "title": campaign["settings"]["title"],
+                        "url": campaign["archive_url"],
+                    }
+                )
+
+        # Check if we've fetched all campaigns
+        total_items = response["total_items"]
+        offset += count
+        if offset >= total_items:
+            break
+
+    # Sort campaigns by send date, newest first
+    campaigns.sort(key=lambda x: x["send_date"], reverse=True)
+    return campaigns
 
+
+class Command(BaseCommand):
+    help = "Fetches newsletters from Mailchimp and saves them to the archive directory"
+
+    def handle(self, *args, **options):
         # Create output directory if it doesn't exist
         output_path = Path(__file__).resolve().parent.parent.parent / "archive"
         output_path.mkdir(parents=True, exist_ok=True)
 
+        cutoff_date = datetime.fromisoformat("2023-07-20")
+
         # Process each newsletter
-        for newsletter in newsletters:
+        for newsletter in fetch_campaign_urls():
             url = newsletter.get("url")
             send_date = newsletter.get("send_date")
             title = newsletter.get("title")
 
             # Convert send_date to a datetime object
             date = datetime.fromisoformat(send_date.replace("Z", "+00:00"))
+
+            # Skip if newsletter is older than cutoff date
+            if date.date() < cutoff_date.date():
+                continue
+
             # Create filename using date and title
             filename = f"{date.strftime('%Y-%m-%d')}_{title.replace(' ', '_')}.html"
             filepath = output_path / filename