Update event notebook

Jupyterhub/clone_conversion_repo.ipynb

@@ -48,9 +48,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cloning disasters-aws-conversion repository...\n",
+      "Successfully cloned disasters-aws-conversion\n"
+     ]
+    }
+   ],
    "source": [
     "import os\n",
     "import subprocess\n",
@@ -74,6 +83,13 @@
     "else:\n",
     "    print(f\"{repo_name} repository already exists\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

Jupyterhub/extract_eventname_metadata.ipynb

@@ -41,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 77,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -59,6 +59,23 @@
     "DELETE_AFTER_MOVE = False  # Set to True to delete original files after successful processing"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Metadata options"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "HAZARD = None\n",
+    "LOCATION = None"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -103,7 +120,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 78,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -143,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 79,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -152,8 +169,8 @@
      "text": [
       "Current AWS Identity:\n",
       "  Account: 515966502221\n",
-      "  ARN: arn:aws:sts::515966502221:assumed-role/disasters-prod/botocore-session-1773693036\n",
-      "  User ID: AROAXQIQAAVGWA2LHXCPW:botocore-session-1773693036\n",
+      "  ARN: arn:aws:sts::515966502221:assumed-role/disasters-prod/botocore-session-1773760456\n",
+      "  User ID: AROAXQIQAAVGWA2LHXCPW:botocore-session-1773760456\n",
       "\n",
       "S3 client created successfully\n"
      ]
@@ -187,7 +204,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 80,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
@@ -329,7 +346,7 @@
     "        return False, {'is_cog': False, 'errors': [str(e)], 'warnings': []}\n",
     "\n",
     "\n",
-    "def add_metadata_and_create_cog(file_bytes: bytes, event_metadata: Dict[str, str]) -> bytes:\n",
+    "def add_metadata_and_create_cog(file_bytes: bytes, event_metadata: Dict[str, str], HAZARD: List[str], LOCATION: List[str], ) -> bytes:\n",
     "    \"\"\"\n",
     "    Rebuild GeoTIFF as a valid COG with metadata using rio-cogeo library.\n",
     "    \n",
@@ -374,11 +391,13 @@
     "        tags = {\n",
     "            'ACTIVATION_EVENT': f\"{event_metadata['year_month']}_{event_metadata['hazard']}_{event_metadata['location']}\",\n",
     "            'YEAR_MONTH': event_metadata['year_month'],\n",
-    "            'HAZARD': event_metadata['hazard'],\n",
-    "            'LOCATION': event_metadata['location'],\n",
+    "            'HAZARD': HAZARD if HAZARD is not None else event_metadata['hazard'],\n",
+    "            'LOCATION': LOCATION if LOCATION is not None else event_metadata['location'],\n",
     "            'PROCESSING_DATE': datetime.utcnow().isoformat(),\n",
     "            'PROCESSOR': 'S3 GeoTIFF COG Processor v1.0'\n",
     "        }\n",
+    "        print(f'Hazard tag {HAZARD}')\n",
+    "        print(f'Location tag {LOCATION}')\n",
     "        \n",
     "        # Create COG profile using original compression settings\n",
     "        cog_profile = {\n",
@@ -408,7 +427,7 @@
     "            dst_path=output_path,\n",
     "            dst_kwargs=cog_profile,\n",
     "            add_mask=False,\n",
-    "            overview_level=4,\n",
+    "            overview_level=5,\n",
     "            overview_resampling='average',\n",
     "            web_optimized=True,\n",
     "            additional_cog_metadata=tags,\n",
@@ -514,7 +533,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 81,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [
     {
@@ -523,6 +542,7 @@
      "text": [
       "Testing pattern detection:\n",
       "\n",
+      "Remainder = Turkey_UNW_2022-04-06_to_2023-02-08_day\n",
       "✓ 202302_Earthquake_Turkiye_Turkey_UNW_2022-04-06_to_2023-02-08_day.tif\n",
       "  Year/Month: 202302\n",
       "  Location: Turkiye\n",
@@ -541,6 +561,7 @@
     "print(\"Testing pattern detection:\\n\")\n",
     "for filename in test_filenames:\n",
     "    result = detect_activation_event(filename)\n",
+    "    print(f'Remainder = {result[\"remainder\"]}')\n",
     "    if result:\n",
     "        print(f\"✓ {filename}\")\n",
     "        print(f\"  Year/Month: {result['year_month']}\")\n",
@@ -561,7 +582,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 82,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [
     {
@@ -603,7 +624,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 83,
+   "execution_count": 30,
    "metadata": {},
    "outputs": [
     {
@@ -633,7 +654,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/tmp/ipykernel_1272/2684847193.py:178: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\n",
+      "/tmp/ipykernel_205/1171966028.py:178: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\n",
       "  'PROCESSING_DATE': datetime.utcnow().isoformat(),\n",
       "Reading input: /vsimem/input.tif\n",
       "\n"
@@ -645,7 +666,9 @@
      "text": [
       "    Reading original file settings...\n",
       "    Original settings: compress=zstd, predictor=None, level=None\n",
-      "    Rebuilding as COG (preserving original compression settings)...\n"
+      "    Rebuilding as COG (preserving original compression settings)...\n",
+      "Hazard tag None\n",
+      "Location tag None\n"
      ]
     },
     {
@@ -662,15 +685,15 @@
      "output_type": "stream",
      "text": [
       "    COG creation complete with embedded metadata\n",
-      "    Output size: 150,638,215 bytes\n",
-      "    Output size: 150,638,215 bytes\n",
-      "    Size change: -4.8%\n",
+      "    Output size: 150,769,442 bytes\n",
+      "    Output size: 150,769,442 bytes\n",
+      "    Size change: -4.7%\n",
       "  Validating output COG structure...\n",
       "    ✓ Output is valid COG\n",
       "      Compression: ZSTD\n",
       "      Tile size: [512, 512]\n",
-      "      Overviews: 4 levels\n",
-      "  Uploading to s3://nasa-disasters-dev/ProgramData/Landsat/burnRatio/202501_Fire_CA_LC09_NBR_182831_041036_2025-01-14_day.tif...\n",
+      "      Overviews: 5 levels\n",
+      "  Uploading to s3://nasa-disasters-dev/ProgramData/Landsat/burnRatio/LC09_NBR_182831_041036_2025-01-14_day.tif...\n",
       "    ✓ Upload successful\n",
       "\n",
       "================================================================================\n",
@@ -697,6 +720,7 @@
     "    \n",
     "    # Step 1: Detect activation event pattern\n",
     "    event_metadata = detect_activation_event(filename)\n",
+    "    final_filename = f'{event_metadata[\"remainder\"]}.tif'\n",
     "    \n",
     "    if not event_metadata:\n",
     "        result['message'] = 'No activation event pattern detected - skipping'\n",
@@ -730,7 +754,7 @@
     "        \n",
     "        # Step 3: Add metadata and create/optimize COG\n",
     "        print(\"  Creating COG with metadata and ZSTD compression...\")\n",
-    "        modified_bytes = add_metadata_and_create_cog(file_bytes, event_metadata)\n",
+    "        modified_bytes = add_metadata_and_create_cog(file_bytes, event_metadata, HAZARD, LOCATION)\n",
     "        print(f\"    Output size: {len(modified_bytes):,} bytes\")\n",
     "        \n",
     "        size_change = ((len(modified_bytes) - len(file_bytes)) / len(file_bytes)) * 100\n",
@@ -754,7 +778,7 @@
     "            result['cog_valid'] = False\n",
     "        \n",
     "        # Step 4: Upload to destination\n",
-    "        dest_key = DESTINATION_PREFIX + filename\n",
+    "        dest_key = DESTINATION_PREFIX + final_filename\n",
     "        print(f\"  Uploading to s3://{DESTINATION_BUCKET}/{dest_key}...\")\n",
     "        \n",
     "        if upload_to_s3(modified_bytes, DESTINATION_BUCKET, dest_key):\n",