Add automated device form and store opsi/puppet data locally (#365)

* Add more automated flow for creating devices

* make department short name optional

* Begin storing provided data locally

* update to django 3

* Improve data loading

* allow device attributes to be archived in description

* Add management command to set OS from mac

* pad default hostname to 6 digit id

* add management command to clean up device names

.gitignore

@@ -21,3 +21,4 @@ backup/
 node_modules/
 /Pipfile
 /Pipfile.lock
+/venv

Lagerregal/base_settings.py

@@ -111,6 +111,7 @@
     'devicetypes',
     'devicegroups',
     'devicetags',
+    'devicedata',
     'locations',
     'users',
     'api',
@@ -190,6 +191,9 @@
 NPM_ROOT_PATH = os.getcwd()
 
 NPM_FILE_PATTERNS = {
+    'alpinejs': [
+        'dist/alpine.js',
+    ],
     'bootstrap': [
         'dist/js/bootstrap.min.js',
     ],
@@ -240,3 +244,9 @@
 TEST_RUNNER = 'Lagerregal.utils.DetectableTestRunner'
 TEST_MODE = False
 PRODUCTION = False
+
+OPERATING_SYSTEMS = [
+    ("win", "Windows"),
+    ("mac", "macOS"),
+    ("linux", "Linux")
+]

Lagerregal/template_production.py

@@ -9,7 +9,7 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'ENGINE': 'django.db.backends.postgresql', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME': '',                      # Or path to database file if using sqlite3.
         'USER': '',                      # Not used with sqlite3.
         'PASSWORD': '',                  # Not used with sqlite3.

Lagerregal/urls.py

@@ -37,6 +37,7 @@
 
     path('devices/', devices_views.DeviceList.as_view(), name="device-list"),
     path('devices/add/', devices_views.DeviceCreate.as_view(), name="device-add"),
+    path('devices/add-automatic/', devices_views.DeviceCreateAutomatic.as_view(), name="device-add-automatic"),
     path('devices/add/template/<int:templateid>/', devices_views.DeviceCreate.as_view(), name="device-add"),
     path('devices/add/copy/<int:copyid>/', devices_views.DeviceCreate.as_view(), name="device-add-copy"),
     path('devices/<int:pk>/', devices_views.DeviceDetail.as_view(), name="device-detail"),
@@ -179,6 +180,7 @@
     path('devices_ajax/devicedetails/<int:device>/', login_required(devicedata_ajax.DeviceDetails.as_view()), name="device-details"),
     path('devices_ajax/devicedetails/<int:device>/json', login_required(devicedata_ajax.DeviceDetailsJson.as_view()), name="device-details-json"),
     path('devices_ajax/devicesoftware/<int:device>/', login_required(devicedata_ajax.DeviceSoftware.as_view()), name="device-software"),
+    path('devices_ajax/initialize_device/', login_required(devices_ajax.InitializeAutomaticDevice.as_view()), name="init-automatic-device"),
 ]
 
 urlpatterns += format_suffix_patterns([

api/serializers.py

@@ -106,6 +106,14 @@ class Meta:
         exclude = ("bookmarkers", )
 
 
+class DeviceIDSerializer(serializers.ModelSerializer):
+    id = serializers.CharField(source="pk", read_only=True)
+
+    class Meta:
+        model = Device
+        exclude = ("bookmarkers", )
+
+
 class DeviceListSerializer(serializers.HyperlinkedModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='device-api-detail')
     id = serializers.CharField(source="pk", read_only=True)

devicedata/ajax.py

@@ -1,32 +1,19 @@
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import get_object_or_404
 from django.shortcuts import render
+from django.utils.timesince import timesince
 from django.views.generic.base import View
 from django.utils.translation import ugettext_lazy as _
 
-from devicedata.providers.opsi import OpsiProvider
-from devicedata.providers.puppet import PuppetProvider
+from devicedata.generic import _get_provider, _update_provided_data
 from devices.models import Device
 import logging
 
-data_providers = {
-    "opsi": OpsiProvider,
-    "puppet": PuppetProvider
-}
+from devices.templatetags.devicetags import as_nested_list
 
 logger = logging.getLogger(__name__)
 
 
-def _get_provider(device):
-    if device.data_provider is not None and device.data_provider in data_providers.keys():
-        return data_providers[device.data_provider]()
-    else:
-        for provider in data_providers.values():
-            provider_instance = provider()
-            if provider_instance.has_device(device):
-                return provider_instance
-
-
 class DeviceDetails(View):
 
     @staticmethod
@@ -40,6 +27,7 @@ def get(request, device):
         context = {
             'device_info': device_info.raw_entries
         }
+        _update_provided_data(device, device_info)
         return render(request, 'devicedata/device_info.html', context)
 
 
@@ -54,9 +42,12 @@ def get(request, device):
         device_info = provider.get_device_info(device)
         raw_entries = [{"name": entry.name,
                         "type": entry.type,
-                        "raw_value": entry.raw_value} for entry in device_info.raw_entries]
+                        "raw_value": as_nested_list(entry.raw_value)} for entry in device_info.raw_entries]
+        new_entries = _update_provided_data(device, device_info)
         formatted_entries = [{"name": entry.name,
-                              "value": entry.value} for entry in device_info.formatted_entries]
+                              "value": entry.formatted_value,
+                              "stored_at": timesince(entry.stored_at)} for entry in new_entries]
+
         return JsonResponse({"raw_entries": raw_entries, "formatted_entries": formatted_entries})
 
 

devicedata/generic.py

@@ -0,0 +1,35 @@
+from django.utils import timezone
+from devicedata.models import ProvidedData
+from devicedata.providers.opsi import OpsiProvider
+from devicedata.providers.puppet import PuppetProvider
+
+data_providers = {
+    "opsi": OpsiProvider,
+    "puppet": PuppetProvider
+}
+
+
+def _get_provider(device):
+    if device.data_provider is not None and device.data_provider in data_providers.keys():
+        return data_providers[device.data_provider]()
+    else:
+        for provider in data_providers.values():
+            provider_instance = provider()
+            if provider_instance.has_device(device):
+                return provider_instance
+
+
+def _update_provided_data(device, data, force=False):
+    if not force:
+        old_data = device.provided_data.all()
+        if len(old_data) > 0:
+            if (timezone.now() - old_data[0].stored_at).days < 7:
+                return old_data
+    device.provided_data.all().delete()
+    for entry in data.formatted_entries:
+        pd = ProvidedData()
+        pd.device = device
+        pd.name = entry.name
+        pd.formatted_value = entry.value
+        pd.save()
+    return device.provided_data.all()

devicedata/management/commands/provider_find.py

@@ -0,0 +1,26 @@
+from django.core.management import BaseCommand
+
+from devicedata.generic import _get_provider
+from devices.models import Device
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument('device_ids', nargs='*', type=int)
+
+    def handle(self, *args, **options):
+        if "device_ids" in options and len(options["device_ids"]) > 0:
+            devices = Device.objects.filter(pk__in=options["device_ids"])
+        else:
+            devices = Device.objects.filter(data_provider="", ipaddress__isnull=False)
+
+        if len(devices) is 0:
+            self.stdout.write("Could not find any devices with data provider.")
+            return
+        for device in devices:
+            provider = _get_provider(device)
+            if provider is not None:
+                device.data_provider = provider.name
+                device.save()
+                self.stdout.write("Processed: {0} with {1}".format(device, device.data_provider))

devicedata/management/commands/provider_import.py

@@ -0,0 +1,30 @@
+from django.core.management import BaseCommand
+
+from devicedata.generic import _get_provider, _update_provided_data
+from devicedata.models import ProvidedData
+from devices.models import Device
+from datetime import datetime
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument('device_ids', nargs='*', type=int)
+        parser.add_argument('-f', '--force', action='store_true', dest='force')
+
+    def handle(self, *args, **options):
+        if "device_ids" in options and len(options["device_ids"]) > 0:
+            devices = Device.objects.filter(pk__in=options["device_ids"])
+        else:
+            devices = Device.objects.exclude(data_provider__isnull=True)
+
+        if len(devices) is 0:
+            self.stdout.write("Could not find any devices with data provider.")
+            return
+        for device in devices:
+            self.stdout.write("Processing: {0} from {1}".format(device, device.data_provider))
+            provider = _get_provider(device)
+            if provider is None:
+                continue
+            data = provider.get_device_info(device)
+            _update_provided_data(device, data, options["force"])

devicedata/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.12 on 2020-05-25 16:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('devices', '0013_auto_20200509_1148'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ProvidedData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('stored_at', models.DateTimeField(auto_created=True)),
+                ('type', models.CharField(max_length=100)),
+                ('name', models.CharField(max_length=200)),
+                ('raw_value', models.CharField(max_length=2000)),
+                ('formatted_value', models.CharField(max_length=500)),
+                ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='provided_data', to='devices.Device')),
+            ],
+        ),
+    ]

devicedata/models.py

@@ -1,3 +1,12 @@
 from django.db import models
+from devices.models import Device
 
-# Create your models here.
+
+class ProvidedData(models.Model):
+    device = models.ForeignKey(Device, on_delete=models.CASCADE, related_name="provided_data")
+
+    type = models.CharField(max_length=100)
+    name = models.CharField(max_length=200)
+    raw_value = models.CharField(max_length=2000)
+    formatted_value = models.CharField(max_length=500)
+    stored_at = models.DateTimeField(auto_now_add=True)

devicedata/providers/base_provider.py

@@ -1,5 +1,4 @@
 from Lagerregal import settings
-from devices.models import Device
 from abc import ABC, abstractmethod
 
 

devicedata/providers/opsi.py

@@ -87,6 +87,7 @@ def format_entries(self):
 
 
 class OpsiProvider(BaseProvider):
+    name = "opsi"
     __connection = OpsiConnection(settings.OPSI_SETTINGS["host"] + ":" + settings.OPSI_SETTINGS["port"],
                                   username=settings.OPSI_SETTINGS["username"],
                                   password=settings.OPSI_SETTINGS["password"],

devicedata/providers/puppet.py

@@ -108,6 +108,7 @@ def format_entries(self):
 
 
 class PuppetProvider(BaseProvider):
+    name = "puppet"
 
     @staticmethod
     def __run_query(query):

devices/ajax.py

@@ -3,7 +3,7 @@
 import urllib
 
 from django.conf import settings
-from django.forms.models import modelform_factory
+from django.forms.models import modelform_factory, model_to_dict
 from django.http import HttpResponse
 from django.http import QueryDict
 from django.shortcuts import get_object_or_404
@@ -234,3 +234,21 @@ def get(self, request):
         return HttpResponse(json.dumps(data), content_type='application/json')
 
 
+class InitializeAutomaticDevice(View):
+
+    def post(self, request):
+        print(request.POST)
+        device = Device()
+        device.name = request.POST["name"]
+        device.department_id = request.POST["department"]
+        device.devicetype_id = request.POST["device_type"]
+        device.operating_system = request.POST["operating_system"]
+        device.creator = self.request.user
+        device.save()
+        device.hostname = "{0}-{1}-{2:06d}".format(device.department.short_name, request.POST["operating_system"], device.pk)
+        device.save()
+
+        return HttpResponse(json.dumps({
+            "id": device.id,
+            "hostname": device.hostname
+        }), content_type='application/json')