Merge branch 'master' into release

.env.example

@@ -88,7 +88,9 @@ EMAIL_HOST_USER=user
 EMAIL_HOST_PASSWORD=password
 DEFAULT_FROM_EMAIL="webmaster@localhost"
 
-QFIELDCLOUD_DEFAULT_NETWORK=qfieldcloud_default
+# Docker compose default network also used by the docker in docker workers
+# If empty value, a default name will be generated at build time, for example `qfieldcloud_default`.
+# QFIELDCLOUD_DEFAULT_NETWORK=""
 
 # Admin URI. Requires slash in the end. Please use something that is hard to guess.
 QFIELDCLOUD_ADMIN_URI=admin/
@@ -122,6 +124,11 @@ QFIELDCLOUD_DEFAULT_TIME_ZONE="Europe/Zurich"
 # DEFAULT: ""
 QFIELDCLOUD_LIBQFIELDSYNC_VOLUME_PATH=""
 
+# QFieldCloud SDK volume path to be mounted by the `worker_wrapper` into `worker` containers.
+# If empty value or invalid value, the pip installed version defined in `requirements_libqfieldsync.txt` will be used.
+# DEFAULT: ""
+QFIELDCLOUD_QFIELDCLOUD_SDK_VOLUME_PATH=""
+
 # The Django development port. Not used in production.
 # DEFAULT: 8011
 DJANGO_DEV_PORT=8011
@@ -162,7 +169,3 @@ DEBUG_DEBUGPY_APP_PORT=5678
 # Debugpy port used for the `worker_wrapper` service
 # DEFAULT: 5679
 DEBUG_DEBUGPY_WORKER_WRAPPER_PORT=5679
-
-# Path to the nginx, letsencrypt, etc configuration files, used by script in `./scripts/`.
-# DEFAULT: ./conf
-CONFIG_PATH=./conf

.gitignore

@@ -13,3 +13,4 @@ conf/certbot/*
 Pipfile*
 **/site-packages
 docker-qgis/libqfieldsync
+docker-qgis/qfieldcloud-sdk-python

docker-app/qfieldcloud/core/admin.py

@@ -19,7 +19,7 @@
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import admin_urlname
 from django.contrib.admin.views.main import ChangeList
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import PermissionDenied, ValidationError
 from django.db.models import Q, QuerySet
 from django.db.models.fields.json import JSONField
 from django.db.models.functions import Lower
@@ -31,6 +31,7 @@
 from django.urls import path, reverse
 from django.utils.decorators import method_decorator
 from django.utils.html import escape, format_html
+from django.utils.http import urlencode
 from django.utils.safestring import SafeText
 from django.utils.translation import gettext_lazy as _
 from django.views.decorators.cache import never_cache
@@ -48,14 +49,15 @@
     Person,
     Project,
     ProjectCollaborator,
+    Secret,
     Team,
     TeamMember,
     User,
     UserAccount,
 )
 from qfieldcloud.core.paginators import LargeTablePaginator
 from qfieldcloud.core.templatetags.filters import filesizeformat10
-from qfieldcloud.core.utils2 import delta_utils, jobs
+from qfieldcloud.core.utils2 import delta_utils, jobs, pg_service_file
 from rest_framework.authtoken.models import TokenProxy
 
 admin.site.unregister(LogEntry)
@@ -115,6 +117,15 @@ def has_delete_permission(self, request, obj=None):
         return super().has_delete_permission(request, obj)
 
 
+class QFieldCloudInlineAdmin(admin.TabularInline):
+    template = "admin/edit_inline/tabular_customized.html"
+
+    def get_formset(self, request, obj=None, **kwargs):
+        self.parent_obj = obj
+
+        return super().get_formset(request, obj, **kwargs)
+
+
 def admin_urlname_by_obj(value, arg):
     if isinstance(value, User):
         if value.is_person:
@@ -584,6 +595,132 @@ def queryset(self, request, queryset):
         return queryset.filter(owner__type=value)
 
 
+class ProjectSecretForm(ModelForm):
+    class Meta:
+        model = Secret
+        fields = ("project", "name", "type", "value", "created_by")
+
+    name = fields.CharField(widget=widgets.TextInput)
+    value = fields.CharField(widget=widgets.Textarea)
+
+    def get_initial_for_field(self, field, field_name):
+        if self.instance.pk and field_name == "value":
+            return ""
+        return super().get_initial_for_field(field, field_name)
+
+    def clean(self):
+        cleaned_data = super().clean()
+
+        if self.instance.pk:
+            type = self.instance.type
+        else:
+            type = cleaned_data.get("type")
+        if type == Secret.Type.PGSERVICE:
+            # validate the pg_service.conf
+            value = cleaned_data.get("value")
+            if value:
+                try:
+                    pg_service_file.validate_pg_service_conf(value)
+                except ValidationError as err:
+                    raise ValidationError({"value": err.message})
+
+            # ensure name with PGSERVICE_SECRET_NAME_PREFIX
+            name = cleaned_data.get("name")
+            if name and not name.startswith(
+                pg_service_file.PGSERVICE_SECRET_NAME_PREFIX
+            ):
+                cleaned_data[
+                    "name"
+                ] = f"{pg_service_file.PGSERVICE_SECRET_NAME_PREFIX}{name}"
+
+        return cleaned_data
+
+
+class SecretAdmin(QFieldCloudModelAdmin):
+    model = Secret
+    form = ProjectSecretForm
+    fields = ("project", "name", "type", "value", "created_by")
+    readonly_fields = ("created_by",)
+    list_display = ("name", "type", "created_by__link", "project__name")
+    autocomplete_fields = ("project",)
+
+    search_fields = (
+        "name__icontains",
+        "project__name__icontains",
+    )
+
+    @admin.display(ordering="created_by", description=_("Created by"))
+    def created_by__link(self, instance):
+        return model_admin_url(instance.created_by)
+
+    @admin.display(ordering="project__name")
+    def project__name(self, instance):
+        return model_admin_url(instance.project, instance.project.name)
+
+    def get_readonly_fields(self, request, obj=None):
+        readonly_fields = super().get_readonly_fields(request, obj)
+
+        if obj:
+            return (*readonly_fields, "name", "type", "project")
+
+        return readonly_fields
+
+    def save_model(self, request, obj, form, change):
+        # only set created_by during the first save
+        if not change:
+            obj.created_by = request.user
+        super().save_model(request, obj, form, change)
+
+    def get_changeform_initial_data(self, request):
+        project_id = request.GET.get("project_id")
+
+        if project_id:
+            project = Project.objects.get(id=project_id)
+        else:
+            project = None
+
+        return {"project": project}
+
+
+class ProjectSecretInline(QFieldCloudInlineAdmin):
+    model = Secret
+    fields = ("link_to_secret", "type", "created_by")
+    readonly_fields = ("link_to_secret",)
+    max_num = 0
+    extra = 0
+
+    @admin.display(description=_("Name"))
+    def link_to_secret(self, obj):
+        url = reverse("admin:core_secret_change", args=[obj.pk])
+        return format_html('<a href="{}">{}</a>', url, obj.name)
+
+    def has_add_permission(self, request, obj=None):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    @property
+    def bottom_html(self):
+        if self.parent_obj:
+            return format_html(
+                """
+                    <a href="{url}?{query_params}" class="btn btn-default form-control">
+                        <i class="fa fa-plus-circle"></i>
+                        {text}
+                    </a>
+                """,
+                url=reverse("admin:core_secret_add"),
+                query_params=urlencode({"project_id": self.parent_obj.pk}),
+                text="Add Secret",
+            )
+        else:
+            return ""
+
+
 class ProjectForm(ModelForm):
     project_files = fields.CharField(
         disabled=True, required=False, widget=ProjectFilesWidget
@@ -642,7 +779,7 @@ class ProjectAdmin(QFieldCloudModelAdmin):
         "data_last_packaged_at",
         "project_details__pre",
     )
-    inlines = (ProjectCollaboratorInline,)
+    inlines = (ProjectCollaboratorInline, ProjectSecretInline)
     search_fields = (
         "id",
         "name__icontains",
@@ -652,6 +789,8 @@ class ProjectAdmin(QFieldCloudModelAdmin):
 
     ordering = ("-updated_at",)
 
+    change_form_template = "admin/project_change_form.html"
+
     def get_form(self, *args, **kwargs):
         help_texts = {
             "file_storage_bytes": _(
@@ -778,6 +917,7 @@ class JobAdmin(QFieldCloudModelAdmin):
         "project__name__iexact",
         "project__owner__username__iexact",
         "id",
+        "project__id__iexact",
     )
     readonly_fields = (
         "project",
@@ -1365,6 +1505,7 @@ class LogEntryAdmin(
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Team, TeamAdmin)
 admin.site.register(Project, ProjectAdmin)
+admin.site.register(Secret, SecretAdmin)
 admin.site.register(Delta, DeltaAdmin)
 admin.site.register(Job, JobAdmin)
 admin.site.register(Geodb, GeodbAdmin)

docker-app/qfieldcloud/core/cron.py

@@ -7,7 +7,7 @@
 from invitations.utils import get_invitation_model
 from sentry_sdk import capture_message
 
-from ..core.models import Job, Project
+from ..core.models import ApplyJob, ApplyJobDelta, Delta, Job, Project
 from ..core.utils2 import storage
 from .invitations_utils import send_invitation
 
@@ -60,6 +60,22 @@ def do(self):
             capture_message(
                 f'Job "{job.id}" was with status "{job.status}", but worker container no longer exists. Job unexpectedly terminated.'
             )
+            if job.type == Job.Type.DELTA_APPLY:
+                ApplyJob.objects.get(id=job.id).deltas_to_apply.update(
+                    last_status=Delta.Status.ERROR,
+                    last_feedback=None,
+                    last_modified_pk=None,
+                    last_apply_attempt_at=job.started_at,
+                    last_apply_attempt_by=job.created_by,
+                )
+
+                ApplyJobDelta.objects.filter(
+                    apply_job_id=job.id,
+                ).update(
+                    status=Delta.Status.ERROR,
+                    feedback=None,
+                    modified_pk=None,
+                )
 
         jobs.update(
             status=Job.Status.FAILED,

docker-app/qfieldcloud/core/templates/admin/edit_inline/tabular_customized.html

@@ -0,0 +1,5 @@
+{% if qfc_admin_inline_included != 1 %}
+    {% include "admin/edit_inline/tabular_extended.html" with qfc_admin_inline_included=1 %}
+{% endif %}
+
+{{ fieldset.opts.bottom_html }}

docker-app/qfieldcloud/core/templates/admin/edit_inline/tabular_extended.html

@@ -0,0 +1 @@
+{% extends "admin/edit_inline/tabular.html" %}

docker-app/qfieldcloud/core/templates/admin/project_change_form.html

@@ -0,0 +1,10 @@
+{% extends 'admin/change_form.html' %}
+{% load i18n %}
+
+{% block submit_buttons_bottom %}
+  {{ block.super }}
+
+  <div class="submit-row">
+    <a href="{% url 'admin:core_job_changelist' %}?q={{original.id}}">{% trans 'Project jobs' %}</a>
+  </div>
+{% endblock %}

docker-app/qfieldcloud/core/templates/admin/project_files_widget.html

@@ -4,10 +4,10 @@
     <dialog class="qfc-admin-project-files-dialog">
         <pre></pre>
 
-        <button type="button">OK</button>
+        <button class="btn btn-primary" type="button">OK</button>
     </dialog>
 
-    <button class="qfc-admin-projects-files-actions-reload-btn" type="button">
+    <button class="qfc-admin-projects-files-actions-reload-btn btn btn-sm btn-info" type="button">
         {% trans 'Refresh Files List' %}
     </button>
     <span class="qfc-admin-project-files-count"></span>
@@ -44,16 +44,16 @@
                         <td></td>
                         <td class="qfc-admin-project-files-text-right"></td>
                         <td>
-                            <button type="button" class="qfc-admin-projects-files-actions-info-btn">{% trans 'Details' %}</button>
+                            <button type="button" class="qfc-admin-projects-files-actions-info-btn btn btn-sm btn-outline-info">{% trans 'Details' %}</button>
                         </td>
                         <td>
-                            <select>
+                            <select class="custom-select-sm">
                                 <option value="">{% trans 'Select a version...' %}</option>
                             </select>
                         </td>
                         <td>
-                            <button type="button" class="qfc-admin-projects-files-actions-download-btn" title="{% trans 'Download the selected version or, by default the latest' %}">{% trans 'Download' %}</button>
-                            <button type="button" class="qfc-admin-projects-files-actions-delete-btn" data-csrf-token="">{% trans 'Delete' %}</button>
+                            <button type="button" class="qfc-admin-projects-files-actions-download-btn btn btn-sm btn-outline-info" title="{% trans 'Download the selected version or, by default the latest' %}">{% trans 'Download' %}</button>
+                            <button type="button" class="qfc-admin-projects-files-actions-delete-btn btn btn-sm btn-outline-danger" data-csrf-token="">{% trans 'Delete' %}</button>
                         </td>
                     </tr>
                 </template>
@@ -68,27 +68,6 @@
         overflow-y: auto;
     }
 
-    .qfc-admin-projects-files-actions-reload-btn,
-    .qfc-admin-projects-files-actions-info-btn,
-    .qfc-admin-projects-files-actions-delete-btn,
-    .qfc-admin-projects-files-actions-download-btn {
-        padding: 8px 15px;
-        border-radius: 4px;
-        cursor: pointer;
-        transition: background 0.15s;
-        vertical-align: middle;
-        font-family: "Roboto", "Lucida Grande", Verdana, Arial, sans-serif;
-        font-weight: normal;
-        font-size: 13px;
-        border: none;
-        background: var(--button-bg);
-        color: var(--button-fg);
-    }
-
-    .qfc-admin-projects-files-actions-delete-btn {
-        background: var(--delete-button-bg);
-    }
-
     .qfc-admin-project-files-modal {
         display: none; /* Hidden by default */
         position: fixed; /* Stay in place */

docker-app/qfieldcloud/core/tests/test_delta.py

@@ -791,7 +791,7 @@ def test_non_spatial_geom_empty_str_delta(self):
 
         self.assertEqual(
             self.get_file_contents(project, "nonspatial.csv"),
-            b'fid,col1\n"1",foo\n"2",newfeature\n',
+            b'fid,col1\n"1",new_value\n',
         )
 
     def test_special_data_types(self):

docker-app/qfieldcloud/core/tests/testdata/delta/deltas/nonspatial_geom_empty_str.json

@@ -12,13 +12,13 @@
             "new": {
                 "geometry": "",
                 "attributes": {
-                    "col1": "foo"
+                    "col1": "new_value"
                 }
             },
             "old": {
                 "geometry": null,
                 "attributes": {
-                    "col1": "bar"
+                    "col1": "foo"
                 }
             }
         }