Introduce pipeline cron service

docker-compose-cron.yaml

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+# Copyright (C) 2025 Collabora Limited
+# Author: Jeny Sadadia <[email protected]>
+
+# version: '3'
+
+services:
+
+  cron:
+    container_name: 'kernelci-pipeline-cron'
+    image: 'kernelci:pipeline-cron'
+    stop_signal: 'SIGINT'
+    restart: on-failure
+    volumes:
+      - './tools/cron/:/home/kernelci/'
+      - './logs/:/home/kernelci/logs/'

requirements-cron.txt

@@ -0,0 +1 @@
+Jinja2==3.1.6

tools/cron/.env

@@ -0,0 +1,9 @@
+UPLOAD_PATH="kci-dev/report"
+FILE_PATH="/home/kernelci/logs/"
+STORAGE_URL="https://files-staging.kernelci.org/"
+STORAGE_TOKEN=<storage-token>
+SMTP_HOST=<smtp-host>
+SMTP_PORT=<smtp-port>
+EMAIL_SENDER=<email-sender>
+EMAIL_PASSWORD=<email-password>
+EMAIL_RECIPIENT=<email-recipient>

tools/cron/email_sender.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+# Copyright (C) 2025 Jeny Sadadia
+# Author: Jeny Sadadia <[email protected]>
+
+"""SMTP Email Sender module"""
+
+from email.mime.multipart import MIMEMultipart
+import email
+import email.mime.text
+import os
+import smtplib
+import sys
+from urllib.parse import urljoin
+import jinja2
+
+
+class EmailSender:
+    """Class to send email report using SMTP"""
+    def __init__(self, smtp_host, smtp_port, email_sender, email_recipient):
+        self._smtp_host = smtp_host
+        self._smtp_port = smtp_port
+        self._email_sender = email_sender
+        self._email_recipient = email_recipient
+        self._email_pass = os.getenv('EMAIL_PASSWORD')
+        template_env = jinja2.Environment(
+                            loader=jinja2.FileSystemLoader(".")
+                        )
+        self._template = template_env.get_template("validation_report_template.jinja2")
+
+    def _smtp_connect(self):
+        """Method to create a connection with SMTP server"""
+        if self._smtp_port == 465:
+            smtp = smtplib.SMTP_SSL(self._smtp_host, self._smtp_port)
+        else:
+            smtp = smtplib.SMTP(self._smtp_host, self._smtp_port)
+            smtp.starttls()
+        smtp.login(self._email_sender, self._email_pass)
+        return smtp
+
+    def _create_email(self, email_subject, email_content):
+        """Method to create an email message from email subject, contect,
+        sender, and receiver"""
+        email_msg = MIMEMultipart()
+        email_text = email.mime.text.MIMEText(email_content, "plain", "utf-8")
+        email_text.replace_header('Content-Transfer-Encoding', 'quopri')
+        email_text.set_payload(email_content, 'utf-8')
+        email_msg.attach(email_text)
+        if isinstance(self._email_recipient, list):
+            email_msg['To'] = ','.join(self._email_recipient)
+        else:
+            email_msg['To'] = self._email_recipient
+        email_msg['From'] = self._email_sender
+        email_msg['Subject'] = email_subject
+        return email_msg
+
+    def _send_email(self, email_msg):
+        """Method to send an email message using SMTP"""
+        smtp = self._smtp_connect()
+        if smtp:
+            smtp.send_message(email_msg)
+            smtp.quit()
+
+    def _get_report(self, report_location, report_url):
+        try:
+            with open(report_location, 'r', encoding='utf-8') as f:
+                report_content = f.read()
+            content = self._template.render(
+                report_content=report_content, report_url=report_url
+            )
+        except Exception as e:
+            print(f"Error reading report file: {e}")
+            sys.exit()
+        return content
+
+    def create_and_send_email(self, email_subject, report_location, report_url):
+        """Method to create and send email"""
+        email_content = self._get_report(report_location, report_url)
+        email_msg = self._create_email(
+            email_subject, email_content
+        )
+        self._send_email(email_msg)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Command line argument missing. Specify report filename.")
+        sys.exit()
+
+    report_filename = sys.argv[1]
+    file_path = os.getenv('FILE_PATH')
+    storage_url = os.getenv('STORAGE_URL')
+    upload_path = os.getenv('UPLOAD_PATH')
+    email_sender = os.getenv('EMAIL_SENDER')
+    email_recipient = os.getenv('EMAIL_RECIPIENT')
+    smtp_host = os.getenv('SMTP_HOST')
+    smtp_port = os.getenv('SMTP_PORT')
+
+    if not any([file_path, storage_url, upload_path,
+                email_sender, email_recipient, smtp_host, smtp_port]):
+        print("Missing environment variables")
+        sys.exit()
+
+    report_url = f"{storage_url+upload_path+'/'+report_filename}"
+    email_sender = EmailSender(
+        smtp_host=smtp_host, smtp_port=smtp_port,
+        email_sender=email_sender,
+        email_recipient=email_recipient,
+    )
+    try:
+        subject = "Maestro Validation report"
+        email_sender.create_and_send_email(
+            email_subject=subject,
+            report_location=urljoin(file_path, report_filename),
+            report_url=report_url,
+        )
+    except Exception as err:
+        print(err)

tools/cron/kci-dev.toml

@@ -0,0 +1,13 @@
+default_instance="production"
+
+[local]
+pipeline="https://127.0.0.1"
+api="http://127.0.0.1:8001/"
+
+[staging]
+pipeline="https://staging.kernelci.org:9100/"
+api="https://staging.kernelci.org:9000/"
+
+[production]
+pipeline="https://kernelci-pipeline.westus3.cloudapp.azure.com/"
+api="https://kernelci-api.westus3.cloudapp.azure.com/"

tools/cron/maestro-validate-cron-job.txt

@@ -0,0 +1,2 @@
+PATH=/usr/local/bin:/usr/bin
+0 0 * * 0 root /home/kernelci/run_maestro_validate.sh

tools/cron/run_maestro_validate.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+timestamp=$(date +"%Y_%m_%d-%H_%M_%S")
+log_file_path="/home/kernelci/logs"
+log_file_name="cron-$timestamp.log"
+cd /home/kernelci
+kci-dev --settings kci-dev.toml maestro validate builds --all-checkouts >> "$log_file_path/$log_file_name"
+kci-dev --settings kci-dev.toml maestro validate boots --all-checkouts >> "$log_file_path/$log_file_name"
+set -a
+source .env
+set +a
+python upload_log.py $log_file_name
+python email_sender.py $log_file_name

tools/cron/upload_log.py

@@ -0,0 +1,33 @@
+import sys
+from urllib.parse import urljoin
+import os
+import requests
+
+
+def upload_file(storage_url, token, upload_path, file_name, file_path):
+    headers = {
+        'Authorization': token,
+    }
+    complete_file_path = urljoin(file_path, file_name)
+    files = {
+        'path': upload_path,
+        'file0': (file_name, open(complete_file_path, "rb").read()),
+    }
+    url = urljoin(storage_url, 'upload')
+    resp = requests.post(url, headers=headers, files=files)
+    resp.raise_for_status()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Command line argument missing. Specify file name to upload.")
+        sys.exit()
+    file_name = sys.argv[1]
+    upload_path = os.getenv("UPLOAD_PATH")
+    file_path = os.getenv("FILE_PATH")
+    storage_url = os.getenv("STORAGE_URL")
+    storage_token = os.getenv("STORAGE_TOKEN")
+    if not any([upload_path, file_path, storage_url, storage_token]):
+        print("Missing environment variables")
+        sys.exit()
+    upload_file(storage_url, storage_token, upload_path, file_name, file_path)

tools/cron/validation_report_template.jinja2

@@ -0,0 +1,10 @@
+Hello,
+
+Please find maestro validation report for this week:
+
+{{ report_content }}
+
+You can also download the report from the URL: {{ report_url }}
+
+Thanks,
+KernelCI team