From 2a0c74d59d9a695d0f3020b7cec15eb2e6222972 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 21:57:55 +0200 Subject: [PATCH 01/19] Add query string to logging output. --- mreg/middleware/logging_http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mreg/middleware/logging_http.py b/mreg/middleware/logging_http.py index 03ff1e27..64597c21 100644 --- a/mreg/middleware/logging_http.py +++ b/mreg/middleware/logging_http.py @@ -106,6 +106,7 @@ def log_request(self, request: HttpRequest) -> None: remote_ip=remote_ip, proxy_ip=proxy_ip, path=request.path_info, + query_string=request.META.get("QUERY_STRING"), request_size=request_size, content=self._get_body(request), ).info("request") @@ -150,6 +151,7 @@ def log_response( status_code=status_code, status_label=status_label, path=request.path_info, + query_string=request.META.get("QUERY_STRING"), content=content, **extra_data, run_time_ms=round(run_time_ms, 2), From 390530fdcbac98cf2e5604850814cbc17eff62e6 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 21:58:22 +0200 Subject: [PATCH 02/19] Type checking fix (type->isinstance) --- mreg/api/v1/tests/tests_bacnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mreg/api/v1/tests/tests_bacnet.py b/mreg/api/v1/tests/tests_bacnet.py index 346b402f..78ec4147 100644 --- a/mreg/api/v1/tests/tests_bacnet.py +++ b/mreg/api/v1/tests/tests_bacnet.py @@ -10,7 +10,7 @@ class BACnetIDTest(MregAPITestCase): basepath = '/api/v1/bacnet/ids/' def basejoin(self, path): - if type(path) != 'str': + if not isinstance(path, str): path = str(path) return self.basepath + path From 1bbc5ce1759ca7aa789675216f3a3d1c337e9891 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 21:59:01 +0200 Subject: [PATCH 03/19] Fix filters for Mreg (HostPolicy outstanding). --- mreg/api/v1/filters.py | 477 ++++++++++++++++++++++++++++++++--------- 1 file changed, 372 insertions(+), 105 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index 78c06a19..c0d5e0e0 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -1,177 +1,444 @@ +from typing import List + from django_filters import rest_framework as filters from mreg.models.base import History from mreg.models.host import BACnetID, Host, HostGroup, Ipaddress, PtrOverride -from mreg.models.network import ( - Label, - NetGroupRegexPermission, - Network, - NetworkExcludedRange, -) -from mreg.models.resource_records import Cname, Hinfo, Loc, Mx, Naptr, Srv, Sshfp, Txt -from mreg.models.zone import ( - ForwardZone, - ForwardZoneDelegation, - NameServer, - ReverseZone, - ReverseZoneDelegation, -) - -class FilterWithID(filters.FilterSet): - id = filters.NumberFilter(field_name="id") - id__in = filters.BaseInFilter(field_name="id") - id__gt = filters.NumberFilter(field_name="id", lookup_expr="gt") - id__lt = filters.NumberFilter(field_name="id", lookup_expr="lt") - - -class JSONFieldExactFilter(filters.CharFilter): - pass +from mreg.models.network import (Label, NetGroupRegexPermission, Network, + NetworkExcludedRange) +from mreg.models.resource_records import (Cname, Hinfo, Loc, Mx, Naptr, Srv, + Sshfp, Txt) +from mreg.models.zone import (ForwardZone, ForwardZoneDelegation, NameServer, + ReverseZone, ReverseZoneDelegation) + +from django.db.models import Q +import operator +from functools import reduce + +import structlog + +mreg_log = structlog.getLogger(__name__) + +OperatorList = List[str] + +STRING_OPERATORS: OperatorList = ["exact", "regex", "contains", "icontains", "startswith", "istartswith", "endswith", "iendswith"] +INT_OPERATORS: OperatorList = ["exact", "in", "gt", "lt"] +EXACT_OPERATORS: OperatorList = ["exact"] class CIDRFieldExactFilter(filters.CharFilter): pass -class BACnetIDFilterSet(FilterWithID): +class BACnetIDFilterSet(filters.FilterSet): class Meta: model = BACnetID - fields = "__all__" - - -class CnameFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + } + +class CnameFilterSet(filters.FilterSet): class Meta: model = Cname - fields = "__all__" - - -class ForwardZoneFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + 'host__name': STRING_OPERATORS, + + 'host__ttl': INT_OPERATORS, + 'name': STRING_OPERATORS, + 'ttl': INT_OPERATORS, + } + +class ForwardZoneFilterSet(filters.FilterSet): class Meta: model = ForwardZone - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "name": STRING_OPERATORS, + } -class ForwardZoneDelegationFilterSet(FilterWithID): +class ForwardZoneDelegationFilterSet(filters.FilterSet): class Meta: model = ForwardZoneDelegation - fields = "__all__" - - -class HinfoFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "name": STRING_OPERATORS, + "nameservers": INT_OPERATORS, + "comment": STRING_OPERATORS, + } + +class HinfoFilterSet(filters.FilterSet): class Meta: model = Hinfo - fields = "__all__" - - -class HistoryFilterSet(FilterWithID): - data = JSONFieldExactFilter(field_name="data") - + fields = { + 'cpu': STRING_OPERATORS, + 'host': INT_OPERATORS, + 'os': STRING_OPERATORS, + } + + +class JSONFieldFilter(filters.CharFilter): + def filter(self, qs, value): + mreg_log.info("JSONFieldFilter called", value=value) + if value: + queries = [Q(**{f"data__{k}": v}) for k, v in self.parent.data.items() if k.startswith('data__')] + return qs.filter(reduce(operator.and_, queries)) + return qs + +class HistoryFilterSet(filters.FilterSet): class Meta: model = History - fields = "__all__" - - -class HostFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "timestamp": INT_OPERATORS, + "user": STRING_OPERATORS, + "resource": STRING_OPERATORS, + "name": STRING_OPERATORS, + "model_id": INT_OPERATORS, + "model": STRING_OPERATORS, + "action": STRING_OPERATORS + } + + # This is a fugly hack to make JSON filtering "work" + def filter_queryset(self, queryset): + data_filters = {k: v for k, v in self.data.items() if k.startswith('data__')} + + if data_filters: + queries = [] + for key, value in data_filters.items(): + json_key = key.split('data__')[1] + if '__in' in json_key: + json_key = json_key.split('__in')[0] + values = value.split(',') + queries.append(Q(**{f"data__{json_key}__in": values})) + else: + queries.append(Q(**{f"data__{json_key}": value})) + + queryset = queryset.filter(reduce(operator.and_, queries)) + + return super().filter_queryset(queryset) + +class HostFilterSet(filters.FilterSet): class Meta: model = Host - fields = "__all__" - - -class HostGroupFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "name": STRING_OPERATORS, + "contact": STRING_OPERATORS, + "ttl": INT_OPERATORS, + "comment": STRING_OPERATORS, + # These are related fields, ie, inverse relationships + "ipaddresses": INT_OPERATORS, + "ipaddresses__ipaddress": EXACT_OPERATORS, + "ipaddresses__macaddress": STRING_OPERATORS, + "ptr_overrides": INT_OPERATORS, + "ptr_overrides__ipaddress": EXACT_OPERATORS, + "hostgroups": INT_OPERATORS, + "hostgroups__name": STRING_OPERATORS, + "hostgroups__description": STRING_OPERATORS, + "bacnetid": INT_OPERATORS, + "mxs": INT_OPERATORS, + "mxs__priority": INT_OPERATORS, + "mxs__mx": STRING_OPERATORS, + "txts": INT_OPERATORS, + "txts__txt": STRING_OPERATORS, + "cnames": INT_OPERATORS, + "cnames__name": STRING_OPERATORS, + "cnames__ttl": INT_OPERATORS, + "naptrs": INT_OPERATORS, + "naptrs__order": INT_OPERATORS, + "naptrs__preference": INT_OPERATORS, + "naptrs__flag": STRING_OPERATORS, + "naptrs__service": STRING_OPERATORS, + "naptrs__regex": STRING_OPERATORS, + "naptrs__replacement": STRING_OPERATORS, + "srvs": INT_OPERATORS, + "srvs__name": STRING_OPERATORS, + "srvs__priority": INT_OPERATORS, + "srvs__weight": INT_OPERATORS, + "srvs__port": INT_OPERATORS, + "srvs__ttl": INT_OPERATORS, + } + + + +class HostGroupFilterSet(filters.FilterSet): class Meta: model = HostGroup - fields = "__all__" - - -class IpaddressFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'description': STRING_OPERATORS, + 'hosts': INT_OPERATORS, + 'name': STRING_OPERATORS, + 'owners': INT_OPERATORS, + 'parent': INT_OPERATORS, + } + + +class IpaddressFilterSet(filters.FilterSet): class Meta: model = Ipaddress - fields = "__all__" - - -class LabelFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + "ipaddress": STRING_OPERATORS, + "macaddress": STRING_OPERATORS, + "host": INT_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__comment": STRING_OPERATORS, + + } + +class LabelFilterSet(filters.FilterSet): class Meta: model = Label - fields = "__all__" + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'description': STRING_OPERATORS, + 'name': STRING_OPERATORS, + } -class LocFilterSet(FilterWithID): +class LocFilterSet(filters.FilterSet): class Meta: model = Loc - fields = "__all__" - - -class MxFilterSet(FilterWithID): + fields = { + 'host': INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'loc': STRING_OPERATORS, + } + + +class MxFilterSet(filters.FilterSet): class Meta: model = Mx - fields = "__all__" - - -class NameServerFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'host': INT_OPERATORS , + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'priority': INT_OPERATORS, + 'mx': STRING_OPERATORS, + } + + +class NameServerFilterSet(filters.FilterSet): class Meta: model = NameServer - fields = "__all__" + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'name': STRING_OPERATORS, + 'ttl': INT_OPERATORS, + } + -class NaptrFilterSet(FilterWithID): +class NaptrFilterSet(filters.FilterSet): class Meta: model = Naptr - fields = "__all__" - - -class NetGroupRegexPermissionFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'host': INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'preference': INT_OPERATORS, + 'order': INT_OPERATORS, + 'flag': STRING_OPERATORS, + 'service': STRING_OPERATORS, + 'regex': STRING_OPERATORS, + 'replacement': STRING_OPERATORS, + } + + +class NetGroupRegexPermissionFilterSet(filters.FilterSet): range = CIDRFieldExactFilter(field_name="range") class Meta: model = NetGroupRegexPermission - fields = "__all__" + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'group': STRING_OPERATORS, + 'regex': STRING_OPERATORS, + 'labels': INT_OPERATORS, + } -class NetworkFilterSet(FilterWithID): +class NetworkFilterSet(filters.FilterSet): network = CIDRFieldExactFilter(field_name="network") class Meta: model = Network - fields = "__all__" - - -class NetworkExcludedRangeFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'description': STRING_OPERATORS, + 'vlan': INT_OPERATORS, + 'dns_delegated': EXACT_OPERATORS, + 'category': STRING_OPERATORS, + 'location': STRING_OPERATORS, + 'frozen': EXACT_OPERATORS, + 'reserved': INT_OPERATORS, + } + + + +class NetworkExcludedRangeFilterSet(filters.FilterSet): class Meta: model = NetworkExcludedRange - fields = "__all__" - - -class PtrOverrideFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'network': INT_OPERATORS, + 'network__description': STRING_OPERATORS, + 'network__vlan': INT_OPERATORS, + 'network__dns_delegated': EXACT_OPERATORS, + 'network__category': STRING_OPERATORS, + 'network__location': STRING_OPERATORS, + 'network__frozen': EXACT_OPERATORS, + 'network__reserved': INT_OPERATORS, + 'start_ip': STRING_OPERATORS, + 'end_ip': STRING_OPERATORS, + } + + +class PtrOverrideFilterSet(filters.FilterSet): class Meta: model = PtrOverride - fields = "__all__" - - -class ReverseZoneFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'host': INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'ipaddress': EXACT_OPERATORS, + } + + +class ReverseZoneFilterSet(filters.FilterSet): network = CIDRFieldExactFilter(field_name="network") class Meta: model = ReverseZone - fields = "__all__" + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'name': STRING_OPERATORS, + } -class ReverseZoneDelegationFilterSet(FilterWithID): +class ReverseZoneDelegationFilterSet(filters.FilterSet): class Meta: model = ReverseZoneDelegation - fields = "__all__" - -class SrvFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'name': STRING_OPERATORS, + 'nameservers': INT_OPERATORS, + 'comment': STRING_OPERATORS, + 'zone': INT_OPERATORS, + 'zone__name': STRING_OPERATORS, + } + + +class SrvFilterSet(filters.FilterSet): class Meta: model = Srv - fields = "__all__" - - -class SshfpFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'host': INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'name': STRING_OPERATORS, + 'priority': INT_OPERATORS, + 'weight': INT_OPERATORS, + 'port': INT_OPERATORS, + 'ttl': INT_OPERATORS, + } + + +class SshfpFilterSet(filters.FilterSet): class Meta: model = Sshfp - fields = "__all__" - - -class TxtFilterSet(FilterWithID): + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'host': INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'algorithm': INT_OPERATORS, + 'hash_type': INT_OPERATORS, + 'fingerprint': STRING_OPERATORS, + } + + +class TxtFilterSet(filters.FilterSet): class Meta: model = Txt - fields = "__all__" + fields = { + 'id': INT_OPERATORS, + 'created_at': INT_OPERATORS, + 'updated_at': INT_OPERATORS, + 'host': INT_OPERATORS, + 'host__comment': STRING_OPERATORS, + 'host__contact': STRING_OPERATORS, + 'host__name': STRING_OPERATORS, + 'host__ttl': INT_OPERATORS, + 'txt': STRING_OPERATORS, + } From 58a9eb1380e4500a44e68322688b8b7ebd64176b Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 22:48:06 +0200 Subject: [PATCH 04/19] Formatting. --- mreg/api/v1/filters.py | 350 +++++++++++++++++++++-------------------- 1 file changed, 178 insertions(+), 172 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index c0d5e0e0..bc3db262 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -1,5 +1,9 @@ +import operator +from functools import reduce from typing import List +import structlog +from django.db.models import Q from django_filters import rest_framework as filters from mreg.models.base import History @@ -11,17 +15,20 @@ from mreg.models.zone import (ForwardZone, ForwardZoneDelegation, NameServer, ReverseZone, ReverseZoneDelegation) -from django.db.models import Q -import operator -from functools import reduce - -import structlog - mreg_log = structlog.getLogger(__name__) OperatorList = List[str] -STRING_OPERATORS: OperatorList = ["exact", "regex", "contains", "icontains", "startswith", "istartswith", "endswith", "iendswith"] +STRING_OPERATORS: OperatorList = [ + "exact", + "regex", + "contains", + "icontains", + "startswith", + "istartswith", + "endswith", + "iendswith", +] INT_OPERATORS: OperatorList = ["exact", "in", "gt", "lt"] EXACT_OPERATORS: OperatorList = ["exact"] @@ -37,11 +44,12 @@ class Meta: "id": INT_OPERATORS, "host": INT_OPERATORS, "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, + "host__ttl": INT_OPERATORS, } + class CnameFilterSet(filters.FilterSet): class Meta: model = Cname @@ -50,15 +58,15 @@ class Meta: "created_at": INT_OPERATORS, "updated_at": INT_OPERATORS, "host": INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - 'host__name': STRING_OPERATORS, - - 'host__ttl': INT_OPERATORS, - 'name': STRING_OPERATORS, - 'ttl': INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "name": STRING_OPERATORS, + "ttl": INT_OPERATORS, } + class ForwardZoneFilterSet(filters.FilterSet): class Meta: model = ForwardZone @@ -82,13 +90,14 @@ class Meta: "comment": STRING_OPERATORS, } + class HinfoFilterSet(filters.FilterSet): class Meta: model = Hinfo fields = { - 'cpu': STRING_OPERATORS, - 'host': INT_OPERATORS, - 'os': STRING_OPERATORS, + "cpu": STRING_OPERATORS, + "host": INT_OPERATORS, + "os": STRING_OPERATORS, } @@ -96,10 +105,15 @@ class JSONFieldFilter(filters.CharFilter): def filter(self, qs, value): mreg_log.info("JSONFieldFilter called", value=value) if value: - queries = [Q(**{f"data__{k}": v}) for k, v in self.parent.data.items() if k.startswith('data__')] + queries = [ + Q(**{f"data__{k}": v}) + for k, v in self.parent.data.items() + if k.startswith("data__") + ] return qs.filter(reduce(operator.and_, queries)) return qs + class HistoryFilterSet(filters.FilterSet): class Meta: model = History @@ -111,28 +125,29 @@ class Meta: "name": STRING_OPERATORS, "model_id": INT_OPERATORS, "model": STRING_OPERATORS, - "action": STRING_OPERATORS + "action": STRING_OPERATORS, } # This is a fugly hack to make JSON filtering "work" def filter_queryset(self, queryset): - data_filters = {k: v for k, v in self.data.items() if k.startswith('data__')} - + data_filters = {k: v for k, v in self.data.items() if k.startswith("data__")} + if data_filters: queries = [] for key, value in data_filters.items(): - json_key = key.split('data__')[1] - if '__in' in json_key: - json_key = json_key.split('__in')[0] - values = value.split(',') + json_key = key.split("data__")[1] + if "__in" in json_key: + json_key = json_key.split("__in")[0] + values = value.split(",") queries.append(Q(**{f"data__{json_key}__in": values})) else: queries.append(Q(**{f"data__{json_key}": value})) - + queryset = queryset.filter(reduce(operator.and_, queries)) - + return super().filter_queryset(queryset) + class HostFilterSet(filters.FilterSet): class Meta: model = Host @@ -178,27 +193,26 @@ class Meta: } - class HostGroupFilterSet(filters.FilterSet): class Meta: model = HostGroup fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'description': STRING_OPERATORS, - 'hosts': INT_OPERATORS, - 'name': STRING_OPERATORS, - 'owners': INT_OPERATORS, - 'parent': INT_OPERATORS, - } + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "description": STRING_OPERATORS, + "hosts": INT_OPERATORS, + "name": STRING_OPERATORS, + "owners": INT_OPERATORS, + "parent": INT_OPERATORS, + } class IpaddressFilterSet(filters.FilterSet): class Meta: model = Ipaddress fields = { - 'id': INT_OPERATORS, + "id": INT_OPERATORS, "ipaddress": STRING_OPERATORS, "macaddress": STRING_OPERATORS, "host": INT_OPERATORS, @@ -206,18 +220,18 @@ class Meta: "host__ttl": INT_OPERATORS, "host__contact": STRING_OPERATORS, "host__comment": STRING_OPERATORS, - } + class LabelFilterSet(filters.FilterSet): class Meta: model = Label fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'description': STRING_OPERATORS, - 'name': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "description": STRING_OPERATORS, + "name": STRING_OPERATORS, } @@ -225,13 +239,12 @@ class LocFilterSet(filters.FilterSet): class Meta: model = Loc fields = { - 'host': INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'loc': STRING_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "loc": STRING_OPERATORS, } @@ -239,17 +252,16 @@ class MxFilterSet(filters.FilterSet): class Meta: model = Mx fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'host': INT_OPERATORS , - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'priority': INT_OPERATORS, - 'mx': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "priority": INT_OPERATORS, + "mx": STRING_OPERATORS, } @@ -257,34 +269,32 @@ class NameServerFilterSet(filters.FilterSet): class Meta: model = NameServer fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'name': STRING_OPERATORS, - 'ttl': INT_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "name": STRING_OPERATORS, + "ttl": INT_OPERATORS, } - class NaptrFilterSet(filters.FilterSet): class Meta: model = Naptr fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'host': INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'preference': INT_OPERATORS, - 'order': INT_OPERATORS, - 'flag': STRING_OPERATORS, - 'service': STRING_OPERATORS, - 'regex': STRING_OPERATORS, - 'replacement': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "preference": INT_OPERATORS, + "order": INT_OPERATORS, + "flag": STRING_OPERATORS, + "service": STRING_OPERATORS, + "regex": STRING_OPERATORS, + "replacement": STRING_OPERATORS, } @@ -294,12 +304,12 @@ class NetGroupRegexPermissionFilterSet(filters.FilterSet): class Meta: model = NetGroupRegexPermission fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'group': STRING_OPERATORS, - 'regex': STRING_OPERATORS, - 'labels': INT_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "group": STRING_OPERATORS, + "regex": STRING_OPERATORS, + "labels": INT_OPERATORS, } @@ -309,37 +319,36 @@ class NetworkFilterSet(filters.FilterSet): class Meta: model = Network fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'description': STRING_OPERATORS, - 'vlan': INT_OPERATORS, - 'dns_delegated': EXACT_OPERATORS, - 'category': STRING_OPERATORS, - 'location': STRING_OPERATORS, - 'frozen': EXACT_OPERATORS, - 'reserved': INT_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "description": STRING_OPERATORS, + "vlan": INT_OPERATORS, + "dns_delegated": EXACT_OPERATORS, + "category": STRING_OPERATORS, + "location": STRING_OPERATORS, + "frozen": EXACT_OPERATORS, + "reserved": INT_OPERATORS, } - class NetworkExcludedRangeFilterSet(filters.FilterSet): class Meta: model = NetworkExcludedRange fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'network': INT_OPERATORS, - 'network__description': STRING_OPERATORS, - 'network__vlan': INT_OPERATORS, - 'network__dns_delegated': EXACT_OPERATORS, - 'network__category': STRING_OPERATORS, - 'network__location': STRING_OPERATORS, - 'network__frozen': EXACT_OPERATORS, - 'network__reserved': INT_OPERATORS, - 'start_ip': STRING_OPERATORS, - 'end_ip': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "network": INT_OPERATORS, + "network__description": STRING_OPERATORS, + "network__vlan": INT_OPERATORS, + "network__dns_delegated": EXACT_OPERATORS, + "network__category": STRING_OPERATORS, + "network__location": STRING_OPERATORS, + "network__frozen": EXACT_OPERATORS, + "network__reserved": INT_OPERATORS, + "start_ip": STRING_OPERATORS, + "end_ip": STRING_OPERATORS, } @@ -347,16 +356,15 @@ class PtrOverrideFilterSet(filters.FilterSet): class Meta: model = PtrOverride fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'host': INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'ipaddress': EXACT_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "ipaddress": EXACT_OPERATORS, } @@ -366,10 +374,10 @@ class ReverseZoneFilterSet(filters.FilterSet): class Meta: model = ReverseZone fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'name': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "name": STRING_OPERATORS, } @@ -377,14 +385,14 @@ class ReverseZoneDelegationFilterSet(filters.FilterSet): class Meta: model = ReverseZoneDelegation fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'name': STRING_OPERATORS, - 'nameservers': INT_OPERATORS, - 'comment': STRING_OPERATORS, - 'zone': INT_OPERATORS, - 'zone__name': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "name": STRING_OPERATORS, + "nameservers": INT_OPERATORS, + "comment": STRING_OPERATORS, + "zone": INT_OPERATORS, + "zone__name": STRING_OPERATORS, } @@ -392,20 +400,19 @@ class SrvFilterSet(filters.FilterSet): class Meta: model = Srv fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'host': INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'name': STRING_OPERATORS, - 'priority': INT_OPERATORS, - 'weight': INT_OPERATORS, - 'port': INT_OPERATORS, - 'ttl': INT_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "name": STRING_OPERATORS, + "priority": INT_OPERATORS, + "weight": INT_OPERATORS, + "port": INT_OPERATORS, + "ttl": INT_OPERATORS, } @@ -413,18 +420,17 @@ class SshfpFilterSet(filters.FilterSet): class Meta: model = Sshfp fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'host': INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'algorithm': INT_OPERATORS, - 'hash_type': INT_OPERATORS, - 'fingerprint': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "algorithm": INT_OPERATORS, + "hash_type": INT_OPERATORS, + "fingerprint": STRING_OPERATORS, } @@ -432,13 +438,13 @@ class TxtFilterSet(filters.FilterSet): class Meta: model = Txt fields = { - 'id': INT_OPERATORS, - 'created_at': INT_OPERATORS, - 'updated_at': INT_OPERATORS, - 'host': INT_OPERATORS, - 'host__comment': STRING_OPERATORS, - 'host__contact': STRING_OPERATORS, - 'host__name': STRING_OPERATORS, - 'host__ttl': INT_OPERATORS, - 'txt': STRING_OPERATORS, + "id": INT_OPERATORS, + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, + "txt": STRING_OPERATORS, } From 7fc5c0750d1a421cf41c4499b46604a4fc6fcac0 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 22:53:07 +0200 Subject: [PATCH 05/19] More formatting cleanups --- mreg/api/v1/filters.py | 45 ++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index bc3db262..7837f47f 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -8,12 +8,20 @@ from mreg.models.base import History from mreg.models.host import BACnetID, Host, HostGroup, Ipaddress, PtrOverride -from mreg.models.network import (Label, NetGroupRegexPermission, Network, - NetworkExcludedRange) -from mreg.models.resource_records import (Cname, Hinfo, Loc, Mx, Naptr, Srv, - Sshfp, Txt) -from mreg.models.zone import (ForwardZone, ForwardZoneDelegation, NameServer, - ReverseZone, ReverseZoneDelegation) +from mreg.models.network import ( + Label, + NetGroupRegexPermission, + Network, + NetworkExcludedRange, +) +from mreg.models.resource_records import Cname, Hinfo, Loc, Mx, Naptr, Srv, Sshfp, Txt +from mreg.models.zone import ( + ForwardZone, + ForwardZoneDelegation, + NameServer, + ReverseZone, + ReverseZoneDelegation, +) mreg_log = structlog.getLogger(__name__) @@ -33,6 +41,18 @@ EXACT_OPERATORS: OperatorList = ["exact"] +class JSONFieldFilter(filters.CharFilter): + def filter(self, qs, value): + if value: + queries = [ + Q(**{f"data__{k}": v}) + for k, v in self.parent.data.items() + if k.startswith("data__") + ] + return qs.filter(reduce(operator.and_, queries)) + return qs + + class CIDRFieldExactFilter(filters.CharFilter): pass @@ -101,19 +121,6 @@ class Meta: } -class JSONFieldFilter(filters.CharFilter): - def filter(self, qs, value): - mreg_log.info("JSONFieldFilter called", value=value) - if value: - queries = [ - Q(**{f"data__{k}": v}) - for k, v in self.parent.data.items() - if k.startswith("data__") - ] - return qs.filter(reduce(operator.and_, queries)) - return qs - - class HistoryFilterSet(filters.FilterSet): class Meta: model = History From ec62bbdc79585951d3a86ef0a9bda077d590bbb3 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 23:23:13 +0200 Subject: [PATCH 06/19] Safer version of JSON filter to avoid SQL injections. --- mreg/api/v1/filters.py | 48 ++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index 7837f47f..c5008797 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -1,3 +1,4 @@ +import json import operator from functools import reduce from typing import List @@ -8,20 +9,12 @@ from mreg.models.base import History from mreg.models.host import BACnetID, Host, HostGroup, Ipaddress, PtrOverride -from mreg.models.network import ( - Label, - NetGroupRegexPermission, - Network, - NetworkExcludedRange, -) -from mreg.models.resource_records import Cname, Hinfo, Loc, Mx, Naptr, Srv, Sshfp, Txt -from mreg.models.zone import ( - ForwardZone, - ForwardZoneDelegation, - NameServer, - ReverseZone, - ReverseZoneDelegation, -) +from mreg.models.network import (Label, NetGroupRegexPermission, Network, + NetworkExcludedRange) +from mreg.models.resource_records import (Cname, Hinfo, Loc, Mx, Naptr, Srv, + Sshfp, Txt) +from mreg.models.zone import (ForwardZone, ForwardZoneDelegation, NameServer, + ReverseZone, ReverseZoneDelegation) mreg_log = structlog.getLogger(__name__) @@ -43,12 +36,27 @@ class JSONFieldFilter(filters.CharFilter): def filter(self, qs, value): - if value: - queries = [ - Q(**{f"data__{k}": v}) - for k, v in self.parent.data.items() - if k.startswith("data__") - ] + if not value: + return qs + + queries = [] + for k, v in self.parent.data.items(): + if k.startswith("data__"): + json_key = k.split("data__", 1)[1] + + if json_key.endswith("__in"): + json_key = json_key[:-4] + try: + values = json.loads(v) + if not isinstance(values, list): + continue + except json.JSONDecodeError: + continue + queries.append(Q(**{f"data__{json_key}__in": values})) + else: + queries.append(Q(**{f"data__{json_key}": v})) + + if queries: return qs.filter(reduce(operator.and_, queries)) return qs From 9253c29c884b4eca5405d2b4f2dcf3d0f8bb0623 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 6 Aug 2024 23:30:45 +0200 Subject: [PATCH 07/19] Refactor, cleanup. --- mreg/api/v1/filters.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index c5008797..ba88b9cc 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -1,4 +1,3 @@ -import json import operator from functools import reduce from typing import List @@ -34,32 +33,6 @@ EXACT_OPERATORS: OperatorList = ["exact"] -class JSONFieldFilter(filters.CharFilter): - def filter(self, qs, value): - if not value: - return qs - - queries = [] - for k, v in self.parent.data.items(): - if k.startswith("data__"): - json_key = k.split("data__", 1)[1] - - if json_key.endswith("__in"): - json_key = json_key[:-4] - try: - values = json.loads(v) - if not isinstance(values, list): - continue - except json.JSONDecodeError: - continue - queries.append(Q(**{f"data__{json_key}__in": values})) - else: - queries.append(Q(**{f"data__{json_key}": v})) - - if queries: - return qs.filter(reduce(operator.and_, queries)) - return qs - class CIDRFieldExactFilter(filters.CharFilter): pass From 6757d93bd037af33a45109ed73748c693e2b5511 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 7 Aug 2024 12:27:37 +0200 Subject: [PATCH 08/19] Hostpolicy filter "fixes", also bump dependencies. - We are stuck on drf 3.14.0 due to https://github.com/encode/django-rest-framework/issues/9358 until https://github.com/encode/django-rest-framework/pull/9483 goes into prod, hopefully 3.15.3. --- hostpolicy/api/v1/views.py | 52 +++++++++++++++++++++++++++----------- requirements-test.txt | 6 ++--- requirements.txt | 8 +++--- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/hostpolicy/api/v1/views.py b/hostpolicy/api/v1/views.py index b7f970e3..2e48b01f 100644 --- a/hostpolicy/api/v1/views.py +++ b/hostpolicy/api/v1/views.py @@ -2,7 +2,8 @@ from rest_framework import status from rest_framework.response import Response -from django_filters import rest_framework as rest_filters +from django_filters import rest_framework as filters +from rest_framework import filters as rest_filters from hostpolicy.api.permissions import IsSuperOrHostPolicyAdminOrReadOnly from hostpolicy.models import HostPolicyAtom, HostPolicyRole @@ -18,39 +19,60 @@ from mreg.mixins import LowerCaseLookupMixin from mreg.models.host import Host -from . import serializers - -# For some reason the name field for filtersets for HostPolicyAtom and HostPolicyRole does -# not support operators (e.g. __contains, __regex) in the same way as other fields. Yes, -# the name field is a LowerCaseCharField, but the operators work fine in mreg proper. -# To resolve this issue, we create custom fields for the filtersets that use the name field. +from mreg.api.v1.filters import STRING_OPERATORS, INT_OPERATORS -class HostPolicyAtomFilterSet(rest_filters.FilterSet): - name__contains = rest_filters.CharFilter(field_name="name", lookup_expr="contains") - name__regex = rest_filters.CharFilter(field_name="name", lookup_expr="regex") +from . import serializers +# Note that related lookups don't work at the moment, so we need to do them explicitly. +class HostPolicyAtomFilterSet(filters.FilterSet): class Meta: model = HostPolicyAtom - fields = "__all__" + fields = { + "name": STRING_OPERATORS, + "create_date": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "description": STRING_OPERATORS, + "roles": INT_OPERATORS, + } -class HostPolicyRoleFilterSet(rest_filters.FilterSet): - name__contains = rest_filters.CharFilter(field_name="name", lookup_expr="contains") - name__regex = rest_filters.CharFilter(field_name="name", lookup_expr="regex") +class HostPolicyRoleFilterSet(filters.FilterSet): + atoms__name__exact = filters.CharFilter(field_name='atoms__name', lookup_expr='exact') + class Meta: model = HostPolicyRole - fields = "__all__" + fields = { + "name": STRING_OPERATORS, + "create_date": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "hosts": INT_OPERATORS, + "atoms": INT_OPERATORS, + "labels": INT_OPERATORS, + } class HostPolicyAtomLogMixin(HistoryLog): log_resource = 'hostpolicy_atom' model = HostPolicyAtom + filter_backends = ( + rest_filters.SearchFilter, + filters.DjangoFilterBackend, + rest_filters.OrderingFilter, + ) + ordering_fields = "__all__" + class HostPolicyRoleLogMixin(HistoryLog): log_resource = 'hostpolicy_role' model = HostPolicyRole + filter_backends = ( + rest_filters.SearchFilter, + filters.DjangoFilterBackend, + rest_filters.OrderingFilter, + ) + ordering_fields = "__all__" class HostPolicyPermissionsListCreateAPIView(M2MPermissions, diff --git a/requirements-test.txt b/requirements-test.txt index 1bd77db6..33462432 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,14 +3,14 @@ djangorestframework==3.14.0 django-auth-ldap==4.8.0 django-logging-json==1.15 django-netfields==1.3.2 -django-filter==24.2 -structlog==24.1.0 +django-filter==24.3 +structlog==24.4.0 rich==13.7.1 gunicorn==22.0.0 idna==3.7 psycopg2-binary==2.9.9 pika==1.3.2 -sentry-sdk==2.3.1 +sentry-sdk==2.12.0 tzdata==2024.1 # Testing framwork tox diff --git a/requirements.txt b/requirements.txt index 190c1b14..b2d47ca9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ -Django==5.0.6 +Django==5.0.8 djangorestframework==3.14.0 django-auth-ldap==4.8.0 django-netfields==1.3.2 -django-filter==24.2 -structlog==24.1.0 +django-filter==24.3 +structlog==24.4.0 rich==13.7.1 gunicorn==22.0.0 idna==3.7 psycopg2-binary==2.9.9 pika==1.3.2 -sentry-sdk==2.3.1 +sentry-sdk==2.12.0 tzdata==2024.1 # For OpenAPI schema generation. uritemplate From 0c442c1c2e1b5a66fd0df2e45378877603c12f7f Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 7 Aug 2024 14:33:58 +0200 Subject: [PATCH 09/19] Refactoring cleanup. --- mreg/api/v1/filters.py | 128 +++++++++++++---------------------------- 1 file changed, 41 insertions(+), 87 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index ba88b9cc..59451293 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -32,7 +32,18 @@ INT_OPERATORS: OperatorList = ["exact", "in", "gt", "lt"] EXACT_OPERATORS: OperatorList = ["exact"] - +HOST_FIELDS = { + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, +} + +CREATED_UPDATED = { + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, +} class CIDRFieldExactFilter(filters.CharFilter): pass @@ -43,11 +54,7 @@ class Meta: model = BACnetID fields = { "id": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, + **HOST_FIELDS, } @@ -56,15 +63,10 @@ class Meta: model = Cname fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "name": STRING_OPERATORS, "ttl": INT_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, } @@ -73,9 +75,8 @@ class Meta: model = ForwardZone fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "name": STRING_OPERATORS, + **CREATED_UPDATED, } @@ -84,11 +85,10 @@ class Meta: model = ForwardZoneDelegation fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "name": STRING_OPERATORS, "nameservers": INT_OPERATORS, "comment": STRING_OPERATORS, + **CREATED_UPDATED, } @@ -141,8 +141,6 @@ class Meta: model = Host fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "name": STRING_OPERATORS, "contact": STRING_OPERATORS, "ttl": INT_OPERATORS, @@ -178,6 +176,7 @@ class Meta: "srvs__weight": INT_OPERATORS, "srvs__port": INT_OPERATORS, "srvs__ttl": INT_OPERATORS, + **CREATED_UPDATED, } @@ -186,13 +185,12 @@ class Meta: model = HostGroup fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "description": STRING_OPERATORS, "hosts": INT_OPERATORS, "name": STRING_OPERATORS, "owners": INT_OPERATORS, "parent": INT_OPERATORS, + **CREATED_UPDATED, } @@ -203,11 +201,7 @@ class Meta: "id": INT_OPERATORS, "ipaddress": STRING_OPERATORS, "macaddress": STRING_OPERATORS, - "host": INT_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__comment": STRING_OPERATORS, + **HOST_FIELDS, } @@ -216,10 +210,9 @@ class Meta: model = Label fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "description": STRING_OPERATORS, "name": STRING_OPERATORS, + **CREATED_UPDATED, } @@ -227,12 +220,8 @@ class LocFilterSet(filters.FilterSet): class Meta: model = Loc fields = { - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "loc": STRING_OPERATORS, + **HOST_FIELDS, } @@ -241,15 +230,10 @@ class Meta: model = Mx fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "priority": INT_OPERATORS, "mx": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED } @@ -258,10 +242,9 @@ class Meta: model = NameServer fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "name": STRING_OPERATORS, "ttl": INT_OPERATORS, + **CREATED_UPDATED, } @@ -270,19 +253,14 @@ class Meta: model = Naptr fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "preference": INT_OPERATORS, "order": INT_OPERATORS, "flag": STRING_OPERATORS, "service": STRING_OPERATORS, "regex": STRING_OPERATORS, "replacement": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, } @@ -293,11 +271,10 @@ class Meta: model = NetGroupRegexPermission fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "group": STRING_OPERATORS, "regex": STRING_OPERATORS, "labels": INT_OPERATORS, + **CREATED_UPDATED, } @@ -308,8 +285,6 @@ class Meta: model = Network fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "description": STRING_OPERATORS, "vlan": INT_OPERATORS, "dns_delegated": EXACT_OPERATORS, @@ -317,6 +292,7 @@ class Meta: "location": STRING_OPERATORS, "frozen": EXACT_OPERATORS, "reserved": INT_OPERATORS, + **CREATED_UPDATED, } @@ -325,8 +301,6 @@ class Meta: model = NetworkExcludedRange fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "network": INT_OPERATORS, "network__description": STRING_OPERATORS, "network__vlan": INT_OPERATORS, @@ -337,6 +311,7 @@ class Meta: "network__reserved": INT_OPERATORS, "start_ip": STRING_OPERATORS, "end_ip": STRING_OPERATORS, + **CREATED_UPDATED, } @@ -345,17 +320,13 @@ class Meta: model = PtrOverride fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "ipaddress": EXACT_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, } + class ReverseZoneFilterSet(filters.FilterSet): network = CIDRFieldExactFilter(field_name="network") @@ -363,9 +334,8 @@ class Meta: model = ReverseZone fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "name": STRING_OPERATORS, + **CREATED_UPDATED, } @@ -374,13 +344,12 @@ class Meta: model = ReverseZoneDelegation fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, "name": STRING_OPERATORS, "nameservers": INT_OPERATORS, "comment": STRING_OPERATORS, "zone": INT_OPERATORS, "zone__name": STRING_OPERATORS, + **CREATED_UPDATED, } @@ -389,18 +358,13 @@ class Meta: model = Srv fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "name": STRING_OPERATORS, "priority": INT_OPERATORS, "weight": INT_OPERATORS, "port": INT_OPERATORS, "ttl": INT_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, } @@ -409,16 +373,11 @@ class Meta: model = Sshfp fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "algorithm": INT_OPERATORS, "hash_type": INT_OPERATORS, "fingerprint": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, } @@ -427,12 +386,7 @@ class Meta: model = Txt fields = { "id": INT_OPERATORS, - "created_at": INT_OPERATORS, - "updated_at": INT_OPERATORS, - "host": INT_OPERATORS, - "host__comment": STRING_OPERATORS, - "host__contact": STRING_OPERATORS, - "host__name": STRING_OPERATORS, - "host__ttl": INT_OPERATORS, "txt": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, } From 9b65eb1647fbe3316a3cdea554e2f6bd1b6c4eb7 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 7 Aug 2024 20:03:34 +0200 Subject: [PATCH 10/19] Skeleton for testing filters. --- mreg/api/v1/tests/test_filtering.py | 67 +++++++++++++++++++++++++++++ requirements-test.txt | 1 + 2 files changed, 68 insertions(+) create mode 100644 mreg/api/v1/tests/test_filtering.py diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py new file mode 100644 index 00000000..7ad0ee25 --- /dev/null +++ b/mreg/api/v1/tests/test_filtering.py @@ -0,0 +1,67 @@ + +from typing import List + +from mreg.models.host import Host +from mreg.models.resource_records import Cname + +from .tests import MregAPITestCase + +from unittest_parametrize import param +from unittest_parametrize import parametrize +from unittest_parametrize import ParametrizedTestCase + + +class FilterTestCase(ParametrizedTestCase, MregAPITestCase): + """Test filtering.""" + + # endpoint, query_key, target, expected_hits + # + # NOTE: The generated hostnames are UNIQUE across every test case! + # The format is: f"{endpoint}{query_key}{i}.example.com".replace("_", "") + # where i is the index of the hostname (and we make three for each test). + @parametrize( + ("endpoint", "query_key", "target", "expected_hits"), + [ + param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), + param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), + param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), + ], + + ) + def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: + """Test filtering on host.""" + + generate_count = 3 + msg_prefix = f"{endpoint} : {query_key} -> {target} => " + + hosts: List[Host] = [] + cnames: List[Cname] = [] + for i in range(generate_count): + hostname = f"{endpoint}{query_key}{i}.example.com".replace("_", "") + hosts.append(Host.objects.create( + name=hostname, + contact="admin@example.com", + ttl=3600, + comment="Test host", + )) + + for i in range(generate_count): + cname = f"cname.{endpoint}{query_key}{i}.example.com".replace("_", "") + cnames.append(Cname.objects.create( + host=hosts[i], + name=cname, + ttl=3600 + )) + + hostname = hosts[0].name + response = self.client.get(f"/api/v1/{endpoint}/?{query_key}={target}") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") + + for host in hosts: + host.delete() + + for cname in cnames: + cname.delete() + diff --git a/requirements-test.txt b/requirements-test.txt index 33462432..e4ddd128 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -16,6 +16,7 @@ tzdata==2024.1 tox pytest pytest-django +unittest_parametrize # These are currently not used, but should be... pylint From 57bdbba81612f3abc0ac992fac9ac183c00ad01e Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 7 Aug 2024 20:06:53 +0200 Subject: [PATCH 11/19] Formatting. --- mreg/api/v1/tests/test_filtering.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index 7ad0ee25..36a8350e 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -1,15 +1,13 @@ from typing import List +from unittest_parametrize import ParametrizedTestCase, param, parametrize + from mreg.models.host import Host from mreg.models.resource_records import Cname from .tests import MregAPITestCase -from unittest_parametrize import param -from unittest_parametrize import parametrize -from unittest_parametrize import ParametrizedTestCase - class FilterTestCase(ParametrizedTestCase, MregAPITestCase): """Test filtering.""" From ef818a40ddae7b6c122517d53742f209a25495dc Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 7 Aug 2024 20:50:38 +0200 Subject: [PATCH 12/19] Add iexact support, add cases. Fix toml. --- mreg/api/v1/filters.py | 1 + mreg/api/v1/tests/test_filtering.py | 4 ++++ pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index 59451293..0a7b72a2 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -21,6 +21,7 @@ STRING_OPERATORS: OperatorList = [ "exact", + "iexact", "regex", "contains", "icontains", diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index 36a8350e..a689fa65 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -23,6 +23,10 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), + param("cnames", "host__name__iexact", "cnameshostnameiexact1.example.com", 1, id="cnames_host__iexact"), + param("cnames", "host__name__startswith", "cnameshostnamestartswith", 3, id="cnames_host__startswith"), + param("cnames", "host__name__endswith", "endswith2.example.com", 1, id="cnames_host__endswith"), + param("cnames", "host__name__regex", "cnameshostnameregex[0-9]", 3, id="cnames_host__regex"), ], ) diff --git a/pyproject.toml b/pyproject.toml index b090b1a0..ff777168 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -[tool.ruff] +[tool.lint.ruff] # https://beta.ruff.rs/docs/rules/ select = ["E", "F"] line-length = 119 From 921885f9b4be23d473e8e6028072d78c63dbbb94 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 7 Aug 2024 20:57:39 +0200 Subject: [PATCH 13/19] Move to ruff formater. --- mreg/api/v1/tests/test_filtering.py | 37 +++++++++++++---------------- pyproject.toml | 23 +++++++++--------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index a689fa65..eda9e739 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -1,4 +1,3 @@ - from typing import List from unittest_parametrize import ParametrizedTestCase, param, parametrize @@ -21,14 +20,13 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): ("endpoint", "query_key", "target", "expected_hits"), [ param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), - param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), - param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), - param("cnames", "host__name__iexact", "cnameshostnameiexact1.example.com", 1, id="cnames_host__iexact"), - param("cnames", "host__name__startswith", "cnameshostnamestartswith", 3, id="cnames_host__startswith"), - param("cnames", "host__name__endswith", "endswith2.example.com", 1, id="cnames_host__endswith"), - param("cnames", "host__name__regex", "cnameshostnameregex[0-9]", 3, id="cnames_host__regex"), + param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), + param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), + param("cnames", "host__name__iexact", "cnameshostnameiexact1.example.com", 1, id="cnames_host__iexact"), + param("cnames", "host__name__startswith", "cnameshostnamestartswith", 3, id="cnames_host__startswith"), + param("cnames", "host__name__endswith", "endswith2.example.com", 1, id="cnames_host__endswith"), + param("cnames", "host__name__regex", "cnameshostnameregex[0-9]", 3, id="cnames_host__regex"), ], - ) def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: """Test filtering on host.""" @@ -40,20 +38,18 @@ def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, ex cnames: List[Cname] = [] for i in range(generate_count): hostname = f"{endpoint}{query_key}{i}.example.com".replace("_", "") - hosts.append(Host.objects.create( - name=hostname, - contact="admin@example.com", - ttl=3600, - comment="Test host", - )) - + hosts.append( + Host.objects.create( + name=hostname, + contact="admin@example.com", + ttl=3600, + comment="Test host", + ) + ) + for i in range(generate_count): cname = f"cname.{endpoint}{query_key}{i}.example.com".replace("_", "") - cnames.append(Cname.objects.create( - host=hosts[i], - name=cname, - ttl=3600 - )) + cnames.append(Cname.objects.create(host=hosts[i], name=cname, ttl=3600)) hostname = hosts[0].name response = self.client.get(f"/api/v1/{endpoint}/?{query_key}={target}") @@ -66,4 +62,3 @@ def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, ex for cname in cnames: cname.delete() - diff --git a/pyproject.toml b/pyproject.toml index ff777168..e06d4672 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,3 @@ -[tool.lint.ruff] -# https://beta.ruff.rs/docs/rules/ -select = ["E", "F"] -line-length = 119 -exclude = [ - "mreg/migrations/", - "hostpolicy/migrations/", - ".tox", -] - [project] name = "mreg" version = "0.0.1" @@ -21,4 +11,15 @@ requires = [ ] [tool.setuptools] -py-modules = ["mreg", "mregsite", "hostpolicy"] \ No newline at end of file +py-modules = ["mreg", "mregsite", "hostpolicy"] + +[tool.ruff] +# https://beta.ruff.rs/docs/rules/ +select = ["E", "F"] +line-length = 119 +exclude = [ + "mreg/migrations/", + "hostpolicy/migrations/", + ".tox", +] + From 0a9478646911228972972bf23d172ff8aec3f626 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Thu, 8 Aug 2024 09:49:09 +0200 Subject: [PATCH 14/19] More tests. - Add ip support. - Add reverse lookups. --- mreg/api/v1/tests/test_filtering.py | 93 +++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index eda9e739..7c2646fc 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -1,12 +1,51 @@ from typing import List +from itertools import chain from unittest_parametrize import ParametrizedTestCase, param, parametrize -from mreg.models.host import Host +from mreg.models.host import Host, Ipaddress from mreg.models.resource_records import Cname from .tests import MregAPITestCase +def create_hosts(name: str, count: int) -> List[Host]: + """Create hosts.""" + + hosts: List[Host] = [] + for i in range(count): + hosts.append(Host.objects.create( + name=f"{name}{i}.example.com".replace("_", ""), + contact="admin@example.com", + ttl=3600, + comment="Test host", + )) + + return hosts + +def create_cnames(hosts: List[Host]) -> List[Cname]: + """Create cnames.""" + + cnames: List[Cname] = [] + for host in hosts: + cnames.append(Cname.objects.create( + host=host, + name=f"cname.{host.name}", + ttl=3600, + )) + + return cnames + +def create_ipaddresses(hosts: List[Host]) -> List[Ipaddress]: + """Create ipaddresses.""" + + ipaddresses: List[Ipaddress] = [] + for i, host in enumerate(hosts): + ipaddresses.append(Ipaddress.objects.create( + host=host, + ipaddress=f"10.0.0.{i}", + )) + + return ipaddresses class FilterTestCase(ParametrizedTestCase, MregAPITestCase): """Test filtering.""" @@ -19,7 +58,28 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): @parametrize( ("endpoint", "query_key", "target", "expected_hits"), [ + # Direct host filtering param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), + param("hosts", "name__contains", "namecontains1", 1, id="hosts_name__contains"), + param("hosts", "name__icontains", "nameicontains2", 1, id="hosts_name__icontains"), + param("hosts", "name__iexact", "nameiexact2.example.com", 1, id="hosts_name__iexact"), + param("hosts", "name__startswith", "namestartswith1", 1, id="hosts_name__startswith"), + param("hosts", "name__endswith", "endswith0.example.com", 1, id="hosts_name__endswith"), + param("hosts", "name_regex", "nameregex[0-9].example.com", 3, id="hosts_name__regex"), + + # Reverse through Ipaddress + param("hosts", "ipaddresses__ipaddress", "10.0.0.1", 1, id="hosts_ipaddresses__ipaddress"), + + # Reverse through Cname + param("hosts", "cnames__name", "cname.hostscnamesname0.example.com", 1, id="hosts_cnames__name"), + param("hosts", "cnames__name__regex", "cname.*regex[0-1].example.com", 2, id="host_cnames__regex"), + param("hosts", "cnames__name__endswith", "with0.example.com", 1, id="hosts_cnames__endswith"), + + # Indirectly through Ipaddress + param("ipaddresses", "host__name", "ipaddresseshostname0.example.com", 1, id="ipaddresses_host__name"), + param("ipaddresses", "host__name__contains", "contains1", 1, id="ipaddresses_host__contains"), + + # Indirectly through Cname param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), param("cnames", "host__name__iexact", "cnameshostnameiexact1.example.com", 1, id="cnames_host__iexact"), @@ -34,31 +94,16 @@ def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, ex generate_count = 3 msg_prefix = f"{endpoint} : {query_key} -> {target} => " - hosts: List[Host] = [] - cnames: List[Cname] = [] - for i in range(generate_count): - hostname = f"{endpoint}{query_key}{i}.example.com".replace("_", "") - hosts.append( - Host.objects.create( - name=hostname, - contact="admin@example.com", - ttl=3600, - comment="Test host", - ) - ) - - for i in range(generate_count): - cname = f"cname.{endpoint}{query_key}{i}.example.com".replace("_", "") - cnames.append(Cname.objects.create(host=hosts[i], name=cname, ttl=3600)) - - hostname = hosts[0].name + hosts = create_hosts(f"{endpoint}{query_key}", generate_count) + cnames = create_cnames(hosts) + ipadresses = create_ipaddresses(hosts) + response = self.client.get(f"/api/v1/{endpoint}/?{query_key}={target}") - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") data = response.json() self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") - for host in hosts: - host.delete() + for obj in chain(ipadresses, cnames, hosts): + obj.delete() + - for cname in cnames: - cname.delete() From 99fee8a2d5a58931d2cbf7bc4bf79b36dcd36b98 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sat, 10 Aug 2024 12:43:11 +0200 Subject: [PATCH 15/19] Hostpolicy filter tests. - Also reformat as per ruff. --- hostpolicy/api/v1/views.py | 11 ++ mreg/api/v1/tests/test_filtering.py | 167 +++++++++++++++++++++++----- 2 files changed, 153 insertions(+), 25 deletions(-) diff --git a/hostpolicy/api/v1/views.py b/hostpolicy/api/v1/views.py index 2e48b01f..6dc4e715 100644 --- a/hostpolicy/api/v1/views.py +++ b/hostpolicy/api/v1/views.py @@ -37,7 +37,18 @@ class Meta: class HostPolicyRoleFilterSet(filters.FilterSet): + # This seems to be required due to the many-to-many relationships? atoms__name__exact = filters.CharFilter(field_name='atoms__name', lookup_expr='exact') + atoms__name__contains = filters.CharFilter(field_name='atoms__name', lookup_expr='contains') + atoms__name__regex = filters.CharFilter(field_name='atoms__name', lookup_expr='regex') + + hosts__name__exact = filters.CharFilter(field_name='hosts__name', lookup_expr='exact') + hosts__name__contains = filters.CharFilter(field_name='hosts__name', lookup_expr='contains') + hosts__name__regex = filters.CharFilter(field_name='hosts__name', lookup_expr='regex') + + labels__name__exact = filters.CharFilter(field_name='labels__name', lookup_expr='exact') + labels__name__contains = filters.CharFilter(field_name='labels__name', lookup_expr='contains') + labels__name__regex = filters.CharFilter(field_name='labels__name', lookup_expr='regex') class Meta: model = HostPolicyRole diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index 7c2646fc..808a8d9b 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -1,52 +1,123 @@ -from typing import List +from typing import List, Tuple from itertools import chain from unittest_parametrize import ParametrizedTestCase, param, parametrize +from hostpolicy.models import HostPolicyAtom, HostPolicyRole +from mreg.models.base import Label from mreg.models.host import Host, Ipaddress from mreg.models.resource_records import Cname from .tests import MregAPITestCase + def create_hosts(name: str, count: int) -> List[Host]: """Create hosts.""" hosts: List[Host] = [] for i in range(count): - hosts.append(Host.objects.create( - name=f"{name}{i}.example.com".replace("_", ""), - contact="admin@example.com", - ttl=3600, - comment="Test host", - )) + hosts.append( + Host.objects.create( + name=f"{name}{i}.example.com".replace("_", ""), + contact="admin@example.com", + ttl=3600, + comment="Test host", + ) + ) return hosts + def create_cnames(hosts: List[Host]) -> List[Cname]: """Create cnames.""" cnames: List[Cname] = [] for host in hosts: - cnames.append(Cname.objects.create( - host=host, - name=f"cname.{host.name}", - ttl=3600, - )) + cnames.append( + Cname.objects.create( + host=host, + name=f"cname.{host.name}", + ttl=3600, + ) + ) return cnames + def create_ipaddresses(hosts: List[Host]) -> List[Ipaddress]: """Create ipaddresses.""" ipaddresses: List[Ipaddress] = [] for i, host in enumerate(hosts): - ipaddresses.append(Ipaddress.objects.create( - host=host, - ipaddress=f"10.0.0.{i}", - )) + ipaddresses.append( + Ipaddress.objects.create( + host=host, + ipaddress=f"10.0.0.{i}", + ) + ) return ipaddresses +def create_labels(name: str, count: int) -> List[Label]: + """Create labels.""" + + labels: List[Label] = [] + for i in range(count): + labels.append( + Label.objects.create( + name=f"{name}{i}".replace("_", ""), + description="Test label", + ) + ) + + return labels + +def create_atoms(name: str, count: int) -> List[HostPolicyAtom]: + """Create atoms.""" + + atoms: List[HostPolicyAtom] = [] + for i in range(count): + atoms.append( + HostPolicyAtom.objects.create( + name=f"{name}{i}".replace("_", ""), + description=f"Test atom {i}", + ) + ) + + return atoms + + +def create_roles( + name: str, hosts: List[Host], atoms: List[HostPolicyAtom], labels: List[Label] +) -> Tuple[List[HostPolicyRole], List[HostPolicyAtom], List[Label]]: + """Create roles.""" + + if not atoms: + atoms = create_atoms(f"{name}atom", len(hosts)) + + if not labels: + labels = create_labels(f"{name}label", len(hosts)) + + if len(hosts) != len(atoms) or len(hosts) != len(labels): + raise ValueError("Hosts, Atoms, and Labels must be the same length.") + + roles: List[HostPolicyRole] = [] + + for i, h in enumerate(hosts): + policy = HostPolicyRole.objects.create( + name=f"{name}host{i}".replace("_", ""), + description="Test role") + + policy.hosts.add(h) + policy.labels.add(labels[i]) + policy.atoms.add(atoms[i]) + + roles.append(policy) + + + return (roles, atoms, labels) + + class FilterTestCase(ParametrizedTestCase, MregAPITestCase): """Test filtering.""" @@ -55,30 +126,24 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): # NOTE: The generated hostnames are UNIQUE across every test case! # The format is: f"{endpoint}{query_key}{i}.example.com".replace("_", "") # where i is the index of the hostname (and we make three for each test). - @parametrize( - ("endpoint", "query_key", "target", "expected_hits"), - [ + @parametrize(("endpoint", "query_key", "target", "expected_hits"), [ # Direct host filtering param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), param("hosts", "name__contains", "namecontains1", 1, id="hosts_name__contains"), param("hosts", "name__icontains", "nameicontains2", 1, id="hosts_name__icontains"), - param("hosts", "name__iexact", "nameiexact2.example.com", 1, id="hosts_name__iexact"), - param("hosts", "name__startswith", "namestartswith1", 1, id="hosts_name__startswith"), + param("hosts", "name__iexact", "hostsnameiexact2.example.com", 1, id="hosts_name__iexact"), + param("hosts", "name__startswith", "hostsnamestartswith1", 1, id="hosts_name__startswith"), param("hosts", "name__endswith", "endswith0.example.com", 1, id="hosts_name__endswith"), param("hosts", "name_regex", "nameregex[0-9].example.com", 3, id="hosts_name__regex"), - # Reverse through Ipaddress param("hosts", "ipaddresses__ipaddress", "10.0.0.1", 1, id="hosts_ipaddresses__ipaddress"), - # Reverse through Cname param("hosts", "cnames__name", "cname.hostscnamesname0.example.com", 1, id="hosts_cnames__name"), param("hosts", "cnames__name__regex", "cname.*regex[0-1].example.com", 2, id="host_cnames__regex"), param("hosts", "cnames__name__endswith", "with0.example.com", 1, id="hosts_cnames__endswith"), - # Indirectly through Ipaddress param("ipaddresses", "host__name", "ipaddresseshostname0.example.com", 1, id="ipaddresses_host__name"), param("ipaddresses", "host__name__contains", "contains1", 1, id="ipaddresses_host__contains"), - # Indirectly through Cname param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), @@ -106,4 +171,56 @@ def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, ex for obj in chain(ipadresses, cnames, hosts): obj.delete() + @parametrize(("endpoint", "query_key", "target", "expected_hits"), [ + param("roles", "name", "roleshost0", 1, id="roles_name"), + param("roles", "name__contains", "roleshost1", 1, id="roles_name__contains"), + param("roles", "name__icontains", "roleshost2", 1, id="roles_name__icontains"), + param("roles", "name__iexact", "roleshost2", 1, id="roles_name__iexact"), + param("roles", "name__startswith", "roleshost1", 1, id="roles_name__startswith"), + param("roles", "name__endswith", "host1", 1, id="roles_name__endswith"), + param("roles", "name__regex", "roleshost[0-1]", 2, id="roles_name__regex"), + + param("roles", "atoms__name__exact", "rolesatomsnameexact1", 1, id="roles_atoms__name__exact"), + param("roles", "atoms__name__contains", "namecontains1", 1, id="roles_atoms__name__contains"), + param("roles", "atoms__name__regex", "nameregex[0-1]", 2, id="roles_atoms__name__regex"), + + param("roles", "hosts__name__exact", "roleshostsnameexact1.example.com", 1, id="roles_hosts__name__exact"), + param("roles", "hosts__name__contains", "namecontains1", 1, id="roles_hosts__name__contains"), + param("roles", "hosts__name__regex", "nameregex[0-1].example.com", 2, id="roles_hosts__name__regex"), + + param("roles", "labels__name__exact", "roleslabelsnameexact1", 1, id="roles_labels__name__exact"), + param("roles", "labels__name__contains", "namecontains1", 1, id="roles_labels__name__contains"), + param("roles", "labels__name__regex", "nameregex[0-1]", 2, id="roles_labels__name__regex"), + + param("atoms", "name", "atomsname0", 1, id="atoms_name"), + param("atoms", "name__contains", "namecontains1", 1, id="atoms_name__contains"), + param("atoms", "name__icontains", "nameicontains2", 1, id="atoms_name__icontains"), + param("atoms", "name__iexact", "atomsnameiexact2", 1, id="atoms_name__iexact"), + param("atoms", "name__startswith", "atomsnamestartswith1", 1, id="atoms_name__startswith"), + param("atoms", "name__endswith", "endswith0", 1, id="atoms_name__endswith"), + param("atoms", "name__regex", "nameregex[0-1]", 2, id="atoms_name__regex"), + + param("atoms", "description", "Test atom 1", 1, id="atoms_description"), + param("atoms", "description__contains", "Test atom", 3, id="atoms_description__contains"), + param("atoms", "description__regex", "Test atom [0-1]", 2, id="atoms_description__regex"), + + ]) + def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: + """Test filtering on hostpolicy.""" + + generate_count = 3 + msg_prefix = f"{endpoint} : {query_key} -> {target} => " + + hosts = create_hosts(f"{endpoint}{query_key}", generate_count) + atoms = create_atoms(f"{endpoint}{query_key}", generate_count) + labels = create_labels(f"{endpoint}{query_key}", generate_count) + + roles, atoms, labels = create_roles(endpoint, hosts, atoms, labels) + + response = self.client.get(f"/api/v1/hostpolicy/{endpoint}/?{query_key}={target}") + self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") + data = response.json() + self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") + for obj in chain(roles, atoms, labels, hosts): + obj.delete() \ No newline at end of file From 8597485a7b7762c144e82d71dab51170859b276b Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sat, 10 Aug 2024 12:57:00 +0200 Subject: [PATCH 16/19] Add a test for filtering on ?id= for hosts. - This should catch the generic issue of filtering on IDs. Ideally we'd do this for every model that supports ID... --- mreg/api/v1/tests/test_filtering.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index 808a8d9b..1dffacb3 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -223,4 +223,24 @@ def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: s self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") for obj in chain(roles, atoms, labels, hosts): + obj.delete() + + def test_filtering_on_host_id(self) -> None: + """Test filtering on host id.""" + + generate_count = 3 + hosts = create_hosts("hosts", generate_count) + + for host in hosts: + with self.subTest(host=host): + id = host.id # type: ignore + msg_prefix = f"hosts : id -> {id} => " + + response = self.client.get(f"/api/v1/hosts/?id={id}") + self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") + data = response.json() + self.assertEqual(data["results"][0]["id"], id, msg=f"{msg_prefix} {data}") + self.assertEqual(data["count"], 1, msg=f"{msg_prefix} {data}") + + for obj in hosts: obj.delete() \ No newline at end of file From 6585336a2656492d74d187a5e8cbf96bb11b11c4 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sat, 10 Aug 2024 12:58:02 +0200 Subject: [PATCH 17/19] Formatting. --- mreg/api/v1/tests/test_filtering.py | 86 ++++++++++++++--------------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index 1dffacb3..8a11588c 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -58,6 +58,7 @@ def create_ipaddresses(hosts: List[Host]) -> List[Ipaddress]: return ipaddresses + def create_labels(name: str, count: int) -> List[Label]: """Create labels.""" @@ -72,6 +73,7 @@ def create_labels(name: str, count: int) -> List[Label]: return labels + def create_atoms(name: str, count: int) -> List[HostPolicyAtom]: """Create atoms.""" @@ -104,17 +106,11 @@ def create_roles( roles: List[HostPolicyRole] = [] for i, h in enumerate(hosts): - policy = HostPolicyRole.objects.create( - name=f"{name}host{i}".replace("_", ""), - description="Test role") - + policy = HostPolicyRole.objects.create(name=f"{name}host{i}".replace("_", ""), description="Test role") policy.hosts.add(h) policy.labels.add(labels[i]) policy.atoms.add(atoms[i]) - roles.append(policy) - - return (roles, atoms, labels) @@ -126,7 +122,9 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): # NOTE: The generated hostnames are UNIQUE across every test case! # The format is: f"{endpoint}{query_key}{i}.example.com".replace("_", "") # where i is the index of the hostname (and we make three for each test). - @parametrize(("endpoint", "query_key", "target", "expected_hits"), [ + @parametrize( + ("endpoint", "query_key", "target", "expected_hits"), + [ # Direct host filtering param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), param("hosts", "name__contains", "namecontains1", 1, id="hosts_name__contains"), @@ -171,40 +169,37 @@ def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, ex for obj in chain(ipadresses, cnames, hosts): obj.delete() - @parametrize(("endpoint", "query_key", "target", "expected_hits"), [ - param("roles", "name", "roleshost0", 1, id="roles_name"), - param("roles", "name__contains", "roleshost1", 1, id="roles_name__contains"), - param("roles", "name__icontains", "roleshost2", 1, id="roles_name__icontains"), - param("roles", "name__iexact", "roleshost2", 1, id="roles_name__iexact"), - param("roles", "name__startswith", "roleshost1", 1, id="roles_name__startswith"), - param("roles", "name__endswith", "host1", 1, id="roles_name__endswith"), - param("roles", "name__regex", "roleshost[0-1]", 2, id="roles_name__regex"), - - param("roles", "atoms__name__exact", "rolesatomsnameexact1", 1, id="roles_atoms__name__exact"), - param("roles", "atoms__name__contains", "namecontains1", 1, id="roles_atoms__name__contains"), - param("roles", "atoms__name__regex", "nameregex[0-1]", 2, id="roles_atoms__name__regex"), - - param("roles", "hosts__name__exact", "roleshostsnameexact1.example.com", 1, id="roles_hosts__name__exact"), - param("roles", "hosts__name__contains", "namecontains1", 1, id="roles_hosts__name__contains"), - param("roles", "hosts__name__regex", "nameregex[0-1].example.com", 2, id="roles_hosts__name__regex"), - - param("roles", "labels__name__exact", "roleslabelsnameexact1", 1, id="roles_labels__name__exact"), - param("roles", "labels__name__contains", "namecontains1", 1, id="roles_labels__name__contains"), - param("roles", "labels__name__regex", "nameregex[0-1]", 2, id="roles_labels__name__regex"), - - param("atoms", "name", "atomsname0", 1, id="atoms_name"), - param("atoms", "name__contains", "namecontains1", 1, id="atoms_name__contains"), - param("atoms", "name__icontains", "nameicontains2", 1, id="atoms_name__icontains"), - param("atoms", "name__iexact", "atomsnameiexact2", 1, id="atoms_name__iexact"), - param("atoms", "name__startswith", "atomsnamestartswith1", 1, id="atoms_name__startswith"), - param("atoms", "name__endswith", "endswith0", 1, id="atoms_name__endswith"), - param("atoms", "name__regex", "nameregex[0-1]", 2, id="atoms_name__regex"), - - param("atoms", "description", "Test atom 1", 1, id="atoms_description"), - param("atoms", "description__contains", "Test atom", 3, id="atoms_description__contains"), - param("atoms", "description__regex", "Test atom [0-1]", 2, id="atoms_description__regex"), - - ]) + @parametrize( + ("endpoint", "query_key", "target", "expected_hits"), + [ + param("roles", "name", "roleshost0", 1, id="roles_name"), + param("roles", "name__contains", "roleshost1", 1, id="roles_name__contains"), + param("roles", "name__icontains", "roleshost2", 1, id="roles_name__icontains"), + param("roles", "name__iexact", "roleshost2", 1, id="roles_name__iexact"), + param("roles", "name__startswith", "roleshost1", 1, id="roles_name__startswith"), + param("roles", "name__endswith", "host1", 1, id="roles_name__endswith"), + param("roles", "name__regex", "roleshost[0-1]", 2, id="roles_name__regex"), + param("roles", "atoms__name__exact", "rolesatomsnameexact1", 1, id="roles_atoms__name__exact"), + param("roles", "atoms__name__contains", "namecontains1", 1, id="roles_atoms__name__contains"), + param("roles", "atoms__name__regex", "nameregex[0-1]", 2, id="roles_atoms__name__regex"), + param("roles", "hosts__name__exact", "roleshostsnameexact1.example.com", 1, id="roles_hosts__name__exact"), + param("roles", "hosts__name__contains", "namecontains1", 1, id="roles_hosts__name__contains"), + param("roles", "hosts__name__regex", "nameregex[0-1].example.com", 2, id="roles_hosts__name__regex"), + param("roles", "labels__name__exact", "roleslabelsnameexact1", 1, id="roles_labels__name__exact"), + param("roles", "labels__name__contains", "namecontains1", 1, id="roles_labels__name__contains"), + param("roles", "labels__name__regex", "nameregex[0-1]", 2, id="roles_labels__name__regex"), + param("atoms", "name", "atomsname0", 1, id="atoms_name"), + param("atoms", "name__contains", "namecontains1", 1, id="atoms_name__contains"), + param("atoms", "name__icontains", "nameicontains2", 1, id="atoms_name__icontains"), + param("atoms", "name__iexact", "atomsnameiexact2", 1, id="atoms_name__iexact"), + param("atoms", "name__startswith", "atomsnamestartswith1", 1, id="atoms_name__startswith"), + param("atoms", "name__endswith", "endswith0", 1, id="atoms_name__endswith"), + param("atoms", "name__regex", "nameregex[0-1]", 2, id="atoms_name__regex"), + param("atoms", "description", "Test atom 1", 1, id="atoms_description"), + param("atoms", "description__contains", "Test atom", 3, id="atoms_description__contains"), + param("atoms", "description__regex", "Test atom [0-1]", 2, id="atoms_description__regex"), + ], + ) def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: """Test filtering on hostpolicy.""" @@ -217,7 +212,7 @@ def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: s roles, atoms, labels = create_roles(endpoint, hosts, atoms, labels) - response = self.client.get(f"/api/v1/hostpolicy/{endpoint}/?{query_key}={target}") + response = self.client.get(f"/api/v1/hostpolicy/{endpoint}/?{query_key}={target}") self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") data = response.json() self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") @@ -233,9 +228,8 @@ def test_filtering_on_host_id(self) -> None: for host in hosts: with self.subTest(host=host): - id = host.id # type: ignore + id = host.id # type: ignore msg_prefix = f"hosts : id -> {id} => " - response = self.client.get(f"/api/v1/hosts/?id={id}") self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") data = response.json() @@ -243,4 +237,4 @@ def test_filtering_on_host_id(self) -> None: self.assertEqual(data["count"], 1, msg=f"{msg_prefix} {data}") for obj in hosts: - obj.delete() \ No newline at end of file + obj.delete() From da19a077083a32ea7a502b76f1d7cec5d6920810 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 12 Aug 2024 07:12:54 +0200 Subject: [PATCH 18/19] Refactor id testing, support `__in`. --- mreg/api/v1/tests/test_filtering.py | 47 +++++++++++++---------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index 8a11588c..fdf70ab5 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -1,4 +1,4 @@ -from typing import List, Tuple +from typing import List, Tuple, Union from itertools import chain from unittest_parametrize import ParametrizedTestCase, param, parametrize @@ -113,7 +113,12 @@ def create_roles( roles.append(policy) return (roles, atoms, labels) - +def resolve_target(target: Union[str, List[str]]) -> List[int]: + if isinstance(target, str): + return [int(target)] + else: + return [int(t) for t in target] + class FilterTestCase(ParametrizedTestCase, MregAPITestCase): """Test filtering.""" @@ -126,13 +131,17 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): ("endpoint", "query_key", "target", "expected_hits"), [ # Direct host filtering + param("hosts", "id", "1", 1, id="hosts_id"), + param("hosts", "id__in", "2", 1, id="hosts_id__in_2"), + param("hosts", "id__in", "1,2", 2, id="hosts_id__in_12"), + param("hosts", "id__in", "0,1,2", 3, id="hosts_id__in_012"), param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), param("hosts", "name__contains", "namecontains1", 1, id="hosts_name__contains"), param("hosts", "name__icontains", "nameicontains2", 1, id="hosts_name__icontains"), param("hosts", "name__iexact", "hostsnameiexact2.example.com", 1, id="hosts_name__iexact"), param("hosts", "name__startswith", "hostsnamestartswith1", 1, id="hosts_name__startswith"), param("hosts", "name__endswith", "endswith0.example.com", 1, id="hosts_name__endswith"), - param("hosts", "name_regex", "nameregex[0-9].example.com", 3, id="hosts_name__regex"), + param("hosts", "name__regex", "nameregex[1-2].example.com", 2, id="hosts_name__regex"), # Reverse through Ipaddress param("hosts", "ipaddresses__ipaddress", "10.0.0.1", 1, id="hosts_ipaddresses__ipaddress"), # Reverse through Cname @@ -148,19 +157,24 @@ class FilterTestCase(ParametrizedTestCase, MregAPITestCase): param("cnames", "host__name__iexact", "cnameshostnameiexact1.example.com", 1, id="cnames_host__iexact"), param("cnames", "host__name__startswith", "cnameshostnamestartswith", 3, id="cnames_host__startswith"), param("cnames", "host__name__endswith", "endswith2.example.com", 1, id="cnames_host__endswith"), - param("cnames", "host__name__regex", "cnameshostnameregex[0-9]", 3, id="cnames_host__regex"), + param("cnames", "host__name__regex", "cnameshostnameregex[0-1]", 2, id="cnames_host__regex"), ], ) def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: """Test filtering on host.""" - generate_count = 3 - msg_prefix = f"{endpoint} : {query_key} -> {target} => " - + generate_count = 3 hosts = create_hosts(f"{endpoint}{query_key}", generate_count) cnames = create_cnames(hosts) ipadresses = create_ipaddresses(hosts) + if query_key.startswith("id"): + targets = target.split(",") if query_key.endswith("__in") else [target] + resolved_targets = resolve_target(targets) + target = ",".join(str(hosts[t].id) for t in resolved_targets) # type: ignore + + msg_prefix = f"{endpoint} : {query_key} -> {target} => " + response = self.client.get(f"/api/v1/{endpoint}/?{query_key}={target}") self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") data = response.json() @@ -219,22 +233,3 @@ def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: s for obj in chain(roles, atoms, labels, hosts): obj.delete() - - def test_filtering_on_host_id(self) -> None: - """Test filtering on host id.""" - - generate_count = 3 - hosts = create_hosts("hosts", generate_count) - - for host in hosts: - with self.subTest(host=host): - id = host.id # type: ignore - msg_prefix = f"hosts : id -> {id} => " - response = self.client.get(f"/api/v1/hosts/?id={id}") - self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") - data = response.json() - self.assertEqual(data["results"][0]["id"], id, msg=f"{msg_prefix} {data}") - self.assertEqual(data["count"], 1, msg=f"{msg_prefix} {data}") - - for obj in hosts: - obj.delete() From bf55f7a79ca9f211a4daedfb0c29029fe06967b3 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 12 Aug 2024 08:59:46 +0200 Subject: [PATCH 19/19] Support CIDR matching. - Match exact CIDR or IP within a CIDR. --- mreg/api/v1/filters.py | 20 +++++++++++++------ mreg/api/v1/tests/test_filtering.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index 0a7b72a2..04741c57 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -15,6 +15,8 @@ from mreg.models.zone import (ForwardZone, ForwardZoneDelegation, NameServer, ReverseZone, ReverseZoneDelegation) +from netaddr import IPNetwork, AddrFormatError + mreg_log = structlog.getLogger(__name__) OperatorList = List[str] @@ -45,10 +47,16 @@ "created_at": INT_OPERATORS, "updated_at": INT_OPERATORS, } +class CIDRFieldFilter(filters.CharFilter): + def filter(self, qs, value): + if not value: + return qs -class CIDRFieldExactFilter(filters.CharFilter): - pass - + try: + cidr = IPNetwork(value) + return qs.filter(**{f"{self.field_name}__net_contains_or_equals": str(cidr)}) + except AddrFormatError: + return qs.none() class BACnetIDFilterSet(filters.FilterSet): class Meta: @@ -266,7 +274,7 @@ class Meta: class NetGroupRegexPermissionFilterSet(filters.FilterSet): - range = CIDRFieldExactFilter(field_name="range") + range = CIDRFieldFilter(field_name="range") class Meta: model = NetGroupRegexPermission @@ -280,7 +288,7 @@ class Meta: class NetworkFilterSet(filters.FilterSet): - network = CIDRFieldExactFilter(field_name="network") + network = CIDRFieldFilter(field_name="network") class Meta: model = Network @@ -329,7 +337,7 @@ class Meta: class ReverseZoneFilterSet(filters.FilterSet): - network = CIDRFieldExactFilter(field_name="network") + network = CIDRFieldFilter(field_name="network") class Meta: model = ReverseZone diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py index fdf70ab5..feb0b1b2 100644 --- a/mreg/api/v1/tests/test_filtering.py +++ b/mreg/api/v1/tests/test_filtering.py @@ -6,6 +6,7 @@ from hostpolicy.models import HostPolicyAtom, HostPolicyRole from mreg.models.base import Label from mreg.models.host import Host, Ipaddress +from mreg.models.network import NetGroupRegexPermission from mreg.models.resource_records import Cname from .tests import MregAPITestCase @@ -233,3 +234,33 @@ def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: s for obj in chain(roles, atoms, labels, hosts): obj.delete() + + @parametrize(("cidr", "exists"), [ + param("10.0.0.0/24", True, id="cidr_0_true"), + param("10.0.1.0/24", True, id="cidr_1_true"), + param("10.0.2.0/24", True, id="cidr_2_true"), + param("10.0.3.0/24", False, id="cidr_3_false"), + + param("10.0.0.1", True, id="ip_0_1_true"), + param("10.0.0.2", True, id="ip_0_2_true"), + param("10.0.1.1", True, id="ip_1_1_true"), + param("10.0.2.1", True, id="ip_2_1_true"), + + param("10.0.3.1", False, id="ip_3_1_false"), + ], + ) + def test_filter_netgroup_regex_permission(self, cidr: str, exists: bool) -> None: + """Test filtering on netgroup regex permission.""" + + generate_count = 3 + + for i in range(generate_count): + NetGroupRegexPermission.objects.create( + regex=".*", + range=f"10.0.{i}.0/24" + ) + + response = self.client.get(f"/api/v1/permissions/netgroupregex/?range={cidr}") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["count"], 1 if exists else 0) \ No newline at end of file