Preload dispvm dom0 feat

.gitlab-ci.yml

@@ -63,6 +63,8 @@ include:
   project: QubesOS/qubes-continuous-integration
 - file: /r4.3/gitlab-host.yml
   project: QubesOS/qubes-continuous-integration
+- file: /r4.3/gitlab-host-openqa.yml
+  project: QubesOS/qubes-continuous-integration
 
 lint:
   extends: .lint

.pylintrc

@@ -58,19 +58,19 @@ const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$
 class-rgx=([A-Z_][a-zA-Z0-9]+|TC_\d\d_[a-zA-Z0-9_]+)$
 
 # Regular expression which should only match correct function names
-function-rgx=[a-z_][a-z0-9_]{2,30}$
+function-rgx=(test_[0-9]{3}_[a-z0-9_]{2,50}|[a-z_][a-z0-9_]{2,40})$
 
 # Regular expression which should only match correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
+method-rgx=(test_[0-9]{3}_[a-z0-9_]{2,50}|[a-z_][a-z0-9_]{2,40})$
 
 # Regular expression which should only match correct instance attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
+attr-rgx=[a-z_][a-z0-9_]{2,40}$
 
 # Regular expression which should only match correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
+argument-rgx=[a-z_][a-z0-9_]{2,40}$
 
 # Regular expression which should only match correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
+variable-rgx=[a-z_][a-z0-9_]{2,40}$
 
 # Regular expression which should only match correct list comprehension /
 # generator expression variable names

Makefile

@@ -179,6 +179,7 @@ all:
 
 install:
 ifeq ($(OS),Linux)
+	$(MAKE) install -C linux/autostart
 	$(MAKE) install -C linux/systemd
 	$(MAKE) install -C linux/aux-tools
 	$(MAKE) install -C linux/system-config

linux/autostart/Makefile

@@ -0,0 +1,6 @@
+all:
+	true
+
+install:
+	mkdir -p $(DESTDIR)/etc/xdg/autostart
+	cp qubes-preload-dispvm.desktop $(DESTDIR)/etc/xdg/autostart

linux/autostart/qubes-preload-dispvm.desktop

@@ -0,0 +1,9 @@
+[Desktop Entry]
+Icon=qubes
+Name=Qubes Preload Disposables
+Comment=Workaround for session monitoring with qubes.WaitForSession
+Categories=System;Monitor;
+Exec=systemctl start qubes-preload-dispvm.service
+Terminal=false
+NoDisplay=true
+Type=Application

linux/aux-tools/Makefile

@@ -3,6 +3,7 @@ all:
 
 install:
 	mkdir -p $(DESTDIR)/usr/lib/qubes
-	cp cleanup-dispvms $(DESTDIR)/usr/lib/qubes
-	cp startup-misc.sh $(DESTDIR)/usr/lib/qubes
+	cp preload-dispvm $(DESTDIR)/usr/lib/qubes/
+	cp cleanup-dispvms $(DESTDIR)/usr/lib/qubes/
+	cp startup-misc.sh $(DESTDIR)/usr/lib/qubes/
 	cp fix-dir-perms.sh $(DESTDIR)/usr/lib/qubes/

linux/aux-tools/preload-dispvm

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import asyncio
+import concurrent.futures
+import qubesadmin
+
+
+def get_max(qube):
+    return int(qube.features.get("preload-dispvm-max", 0) or 0)
+
+
+async def main():
+    domains = qubesadmin.Qubes().domains
+    appvms = [
+        qube
+        for qube in domains
+        if get_max(qube) > 0
+        and qube.klass == "AppVM"
+        and getattr(qube, "template_for_dispvms", False)
+    ]
+    method = "admin.vm.CreateDisposable"
+    loop = asyncio.get_running_loop()
+    tasks = []
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        for qube in appvms:
+            maximum = get_max(qube)
+            print(f"{qube}:{maximum}")
+            exec_args = qube.qubesd_call, qube.name, method, "preload-autostart"
+            future = loop.run_in_executor(executor, *exec_args)
+            tasks.append(future)
+        await asyncio.gather(*tasks)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

linux/systemd/Makefile

@@ -9,6 +9,7 @@ install:
 	cp [email protected] $(DESTDIR)$(UNITDIR)
 	cp qubes-qmemman.service $(DESTDIR)$(UNITDIR)
 	cp qubesd.service $(DESTDIR)$(UNITDIR)
+	cp qubes-preload-dispvm.service $(DESTDIR)$(UNITDIR)
 	install -d $(DESTDIR)$(UNITDIR)/[email protected]
 	install -m 0644 [email protected]_30_qubes.conf \
 		$(DESTDIR)$(UNITDIR)/[email protected]/30_qubes.conf

linux/systemd/qubes-preload-dispvm.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Preload Qubes DispVMs
+ConditionKernelCommandLine=!qubes.skip_autostart
+# After qmemman so the daemon can create the file containing available memory.
+After=qubesd.service qubes-meminfo-writer-dom0.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/lib/qubes/preload-dispvm
+Group=qubes
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

linux/systemd/qubes-vm@.service

@@ -1,12 +1,12 @@
 [Unit]
 Description=Start Qubes VM %i
 After=qubesd.service qubes-meminfo-writer-dom0.service
+Before=qubes-preload-dispvm.service
 ConditionKernelCommandLine=!qubes.skip_autostart
 
 [Service]
 Type=oneshot
-Environment=DISPLAY=:0
-ExecStart=/usr/bin/qvm-start --skip-if-running %i
+ExecStart=/usr/bin/qvm-start --skip-if-running -- %i
 Group=qubes
 RemainAfterExit=yes
 

qubes/api/admin.py

@@ -24,9 +24,10 @@
 import asyncio
 import functools
 import os
+import pathlib
+import re
 import string
 import subprocess
-import pathlib
 
 from ctypes import CDLL
 
@@ -1159,9 +1160,18 @@ async def vm_feature_remove(self):
 
     @qubes.api.method("admin.vm.feature.Set", scope="local", write=True)
     async def vm_feature_set(self, untrusted_payload):
-        # validation of self.arg done by qrexec-policy is enough
-        value = untrusted_payload.decode("ascii", errors="strict")
+        untrusted_value = untrusted_payload.decode("ascii", errors="strict")
         del untrusted_payload
+        if re.match(r"\A[a-zA-Z0-9_.-]+\Z", self.arg) is None:
+            raise qubes.exc.QubesValueError(
+                "feature name contains illegal characters"
+            )
+        if re.match(r"\A[\x20-\x7E]*\Z", untrusted_value) is None:
+            raise qubes.exc.QubesValueError(
+                f"{self.arg} value contains illegal characters"
+            )
+        value = untrusted_value
+        del untrusted_value
 
         self.fire_event_for_permission(value=value)
         self.dest.features[self.arg] = value
@@ -1296,25 +1306,33 @@ async def _vm_create(
 
     @qubes.api.method("admin.vm.CreateDisposable", scope="global", write=True)
     async def create_disposable(self, untrusted_payload):
-        self.enforce(not self.arg)
+        """
+        Create a disposable. If the RPC argument is ``preload-autostart``,
+        cleanse the preload list and start preloading fresh disposables.
+        """
+        self.enforce(self.arg in [None, "", "preload-autostart"])
+        preload_autostart = False
+        if self.arg == "preload-autostart":
+            preload_autostart = True
         if untrusted_payload not in (b"", b"uuid"):
             raise qubes.exc.QubesValueError(
                 "Invalid payload for admin.vm.CreateDisposable: "
                 "expected the empty string or 'uuid'"
             )
 
         if self.dest.name == "dom0":
-            dispvm_template = self.src.default_dispvm
+            appvm = self.src.default_dispvm
         else:
-            dispvm_template = self.dest
+            appvm = self.dest
 
-        self.fire_event_for_permission(dispvm_template=dispvm_template)
-
-        dispvm = await qubes.vm.dispvm.DispVM.from_appvm(dispvm_template)
+        self.fire_event_for_permission(dispvm_template=appvm)
+        if preload_autostart:
+            await appvm.fire_event_async("domain-preload-dispvm-autostart")
+            return
+        dispvm = await qubes.vm.dispvm.DispVM.from_appvm(appvm)
         # TODO: move this to extension (in race-free fashion, better than here)
         dispvm.tags.add("created-by-" + str(self.src))
         dispvm.tags.add("disp-created-by-" + str(self.src))
-
         return (
             ("uuid:" + str(dispvm.uuid)) if untrusted_payload else dispvm.name
         )
@@ -1659,7 +1677,10 @@ async def vm_device_set_required(self, endpoint, untrusted_payload):
         self.app.save()
 
     @qubes.api.method(
-        "admin.vm.device.denied.List", no_payload=True, scope="local", read=True
+        "admin.vm.device.denied.List",
+        no_payload=True,
+        scope="local",
+        read=True,
     )
     async def vm_device_denied_list(self):
         """
@@ -1767,7 +1788,10 @@ async def vm_firewall_set(self, untrusted_payload):
         self.dest.firewall.save()
 
     @qubes.api.method(
-        "admin.vm.firewall.Reload", no_payload=True, scope="local", execute=True
+        "admin.vm.firewall.Reload",
+        no_payload=True,
+        scope="local",
+        execute=True,
     )
     async def vm_firewall_reload(self):
         self.enforce(not self.arg)

qubes/api/internal.py

@@ -18,7 +18,7 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, see <https://www.gnu.org/licenses/>.
 
-""" Internal interface for dom0 components to communicate with qubesd. """
+"""Internal interface for dom0 components to communicate with qubesd."""
 
 import asyncio
 import json
@@ -45,6 +45,8 @@ class SystemInfoCache:
         "domain-shutdown",
         "domain-tag-add:*",
         "domain-tag-delete:*",
+        "domain-feature-set:internal",
+        "domain-feature-delete:internal",
         "property-set:template_for_dispvms",
         "property-reset:template_for_dispvms",
         "property-set:default_dispvm",
@@ -117,6 +119,7 @@ def get_system_info(cls, app):
         system_info = {
             "domains": {
                 domain.name: {
+                    "internal": domain.features.get("internal", None),
                     "tags": list(domain.tags),
                     "type": domain.__class__.__name__,
                     "template_for_dispvms": getattr(
@@ -262,6 +265,12 @@ async def suspend_pre(self):
         :return:
         """
 
+        preload_templates = qubes.vm.dispvm.get_preload_templates(
+            self.app.domains
+        )
+        for qube in preload_templates:
+            qube.remove_preload_excess(0)
+
         # first keep track of VMs which were paused before suspending
         previously_paused = [
             vm.name
@@ -406,3 +415,11 @@ async def suspend_post(self):
                         qubes.config.suspend_timeout,
                         "qubes.SuspendPostAll",
                     )
+
+        preload_templates = qubes.vm.dispvm.get_preload_templates(
+            self.app.domains
+        )
+        for qube in preload_templates:
+            asyncio.ensure_future(
+                qube.fire_event_async("domain-preload-dispvm-autostart")
+            )

qubes/app.py

@@ -1732,9 +1732,12 @@ def on_property_set_default_netvm(
                     "property-reset:netvm", name="netvm", oldvalue=oldvalue
                 )
 
-    @qubes.events.handler("property-set:default_dispvm")
+    @qubes.events.handler(
+        "property-set:default_dispvm",
+        "property-reset:default_dispvm",
+    )
     def on_property_set_default_dispvm(
-        self, event, name, newvalue, oldvalue=None
+        self, event, name, newvalue=None, oldvalue=None
     ):
         # pylint: disable=unused-argument
         for vm in self.domains:
@@ -1749,6 +1752,25 @@ def on_property_set_default_dispvm(
                     oldvalue=oldvalue,
                 )
 
+        if not newvalue:
+            newvalue = self.default_dispvm
+        if newvalue == oldvalue:
+            return
+        old_appvm = None
+        appvm = None
+        if oldvalue:
+            old_appvm = self.domains[oldvalue]
+        if newvalue:
+            appvm = self.domains[newvalue]
+            appvm.preload_max_ignore_global = False
+        if oldvalue and not newvalue:
+            old_appvm.preload_max_ignore_global = True
+        fire_appvm = [qube for qube in [appvm, old_appvm] if qube is not None]
+        for qube in fire_appvm:
+            asyncio.ensure_future(
+                qube.fire_event_async("domain-preload-dispvm-start")
+            )
+
     @qubes.events.handler("property-pre-set:default_kernel")
     # pylint: disable-next=invalid-name
     def on_property_pre_set_default_kernel(

qubes/config.py

@@ -106,3 +106,7 @@ class Defaults(TypedDict):
 qubes_ipv6_prefix = "fd09:24ef:4179:0000"
 
 suspend_timeout = 60
+
+#: amount of available memory on the system. Beware that the use of a file is
+# subject to change.
+qmemman_avail_mem_file = "/var/run/qubes/qmemman-avail-mem"

qubes/ext/gui.py

@@ -93,6 +93,14 @@ def on_domain_qdb_create(self, vm, event):
             except KeyError:
                 pass
 
+        vm.untrusted_qdb.write(
+            "/qubes-gui-enabled",
+            str(
+                bool(
+                    getattr(vm, "guivm", None) and vm.features.get("gui", True)
+                )
+            ),
+        )
         # Add GuiVM Xen ID for gui-daemon
         if getattr(vm, "guivm", None):
             if vm != vm.guivm:
@@ -129,6 +137,10 @@ async def on_domain_start(self, vm, event, **kwargs):
             domain for domain in self.attached_vms(vm) if domain.is_running()
         ]
         for attached_vm in attached_vms:
+            attached_vm.untrusted_qdb.write(
+                "/qubes-gui-enabled",
+                str(bool(attached_vm.features.get("gui", True))),
+            )
             attached_vm.untrusted_qdb.write(
                 "/qubes-gui-domain-xid", str(vm.xid)
             )

qubes/log.py

@@ -33,7 +33,7 @@ def __init__(self, *args, debug=False, **kwargs):
         self.debug = debug
 
     def formatMessage(self, record):
-        fmt = ""
+        fmt = "%(levelname)s: "
         if self.debug:
             fmt += "[%(processName)s %(module)s.%(funcName)s:%(lineno)d] "
         if self.debug or record.name.startswith("vm."):