diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 75de67dc2..9c82a40b1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ Release notes ============= +Version v33.4.0 +---------------- + +- We added importer specific improvers and removed default improver + additionally improve recent advisories first. + + Version v33.3.0 ---------------- diff --git a/setup.cfg b/setup.cfg index 4231acd33..90adb9f43 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vulnerablecode -version = 33.3.0 +version = 33.4.0 license = Apache-2.0 AND CC-BY-SA-4.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 3d754d6df..04270059a 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -138,7 +138,7 @@ def fetch_advisory_links(self, url): for tag in soup.find_all("a"): link = tag.get("href") - if "security-" in link and any(char.isdigit() for char in link): + if link and "security-" in link and any(char.isdigit() for char in link): yield urllib.parse.urljoin(url, link) def advisory_data(self): diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 629ece67f..dd17f2f4e 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -7,11 +7,38 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from vulnerabilities.improvers import default +from vulnerabilities.improvers import importer_specific_improver from vulnerabilities.improvers import valid_versions IMPROVERS_REGISTRY = [ - default.DefaultImprover, + importer_specific_improver.NVDImprover, + importer_specific_improver.DebianImprover, + importer_specific_improver.DebianOvalImprover, + importer_specific_improver.AlpineLinuxImprover, + importer_specific_improver.ApacheHTTPDImprover, + importer_specific_improver.ApacheKafkaImprover, + importer_specific_improver.ApacheTomcatImprover, + importer_specific_improver.ArchLinuxImprover, + importer_specific_improver.ElixirSecurityImprover, + importer_specific_improver.FireEyeImprover, + importer_specific_improver.GentooImprover, + importer_specific_improver.GitHubAPIImprover, + importer_specific_improver.GitLabAPIImprover, + importer_specific_improver.IstioImprover, + importer_specific_improver.MozillaImprover, + importer_specific_improver.NginxImprover, + importer_specific_improver.NpmImprover, + importer_specific_improver.OpensslImprover, + importer_specific_improver.PostgreSQLImprover, + importer_specific_improver.ProjectKBMSRImprover, + importer_specific_improver.PyPaImprover, + importer_specific_improver.PyPIImprover, + importer_specific_improver.RedhatImprover, + importer_specific_improver.RetireDotnetImprover, + importer_specific_improver.SUSESeverityScoreImprover, + importer_specific_improver.UbuntuImprover, + importer_specific_improver.UbuntuUSNImprover, + importer_specific_improver.XenImprover, valid_versions.NginxBasicImprover, valid_versions.ApacheHTTPDImprover, valid_versions.DebianBasicImprover, diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index 2a006e07b..03cd4abf8 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -12,11 +12,13 @@ from typing import List from typing import Tuple +from django.db.models import Q from django.db.models.query import QuerySet from packageurl import PackageURL from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Importer from vulnerabilities.improver import MAX_CONFIDENCE from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference @@ -34,10 +36,17 @@ class DefaultImprover(Improver): information source. """ + importer: Importer + @property def interesting_advisories(self) -> QuerySet: - for advisory in Advisory.objects.all().paginated(): - yield advisory + if hasattr(self, "importer"): + return ( + Advisory.objects.filter(Q(created_by=self.importer.qualified_name)) + .order_by("-date_collected") + .paginated() + ) + return Advisory.objects.all().order_by("-date_collected").paginated() def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: if not advisory_data: diff --git a/vulnerabilities/improvers/importer_specific_improver.py b/vulnerabilities/improvers/importer_specific_improver.py new file mode 100644 index 000000000..e5aa4a482 --- /dev/null +++ b/vulnerabilities/improvers/importer_specific_improver.py @@ -0,0 +1,150 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from vulnerabilities.importers.alpine_linux import AlpineImporter +from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter +from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter +from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter +from vulnerabilities.importers.archlinux import ArchlinuxImporter +from vulnerabilities.importers.debian import DebianImporter +from vulnerabilities.importers.debian_oval import DebianOvalImporter +from vulnerabilities.importers.elixir_security import ElixirSecurityImporter +from vulnerabilities.importers.fireeye import FireyeImporter +from vulnerabilities.importers.gentoo import GentooImporter +from vulnerabilities.importers.github import GitHubAPIImporter +from vulnerabilities.importers.gitlab import GitLabAPIImporter +from vulnerabilities.importers.istio import IstioImporter +from vulnerabilities.importers.mozilla import MozillaImporter +from vulnerabilities.importers.nginx import NginxImporter +from vulnerabilities.importers.npm import NpmImporter +from vulnerabilities.importers.nvd import NVDImporter +from vulnerabilities.importers.openssl import OpensslImporter +from vulnerabilities.importers.postgresql import PostgreSQLImporter +from vulnerabilities.importers.project_kb_msr2019 import ProjectKBMSRImporter +from vulnerabilities.importers.pypa import PyPaImporter +from vulnerabilities.importers.pysec import PyPIImporter +from vulnerabilities.importers.redhat import RedhatImporter +from vulnerabilities.importers.retiredotnet import RetireDotnetImporter +from vulnerabilities.importers.suse_scores import SUSESeverityScoreImporter +from vulnerabilities.importers.ubuntu import UbuntuImporter +from vulnerabilities.importers.ubuntu_usn import UbuntuUSNImporter +from vulnerabilities.importers.xen import XenImporter +from vulnerabilities.improvers.default import DefaultImprover + + +class NVDImprover(DefaultImprover): + importer = NVDImporter + + +class AlpineLinuxImprover(DefaultImprover): + importer = AlpineImporter + + +class ApacheHTTPDImprover(DefaultImprover): + importer = ApacheHTTPDImporter + + +class ApacheKafkaImprover(DefaultImprover): + importer = ApacheKafkaImporter + + +class ApacheTomcatImprover(DefaultImprover): + importer = ApacheTomcatImporter + + +class ArchLinuxImprover(DefaultImprover): + importer = ArchlinuxImporter + + +class DebianImprover(DefaultImprover): + importer = DebianImporter + + +class DebianOvalImprover(DefaultImprover): + importer = DebianOvalImporter + + +class ElixirSecurityImprover(DefaultImprover): + importer = ElixirSecurityImporter + + +class FireEyeImprover(DefaultImprover): + importer = FireyeImporter + + +class GentooImprover(DefaultImprover): + importer = GentooImporter + + +class GitHubAPIImprover(DefaultImprover): + importer = GitHubAPIImporter + + +class GitLabAPIImprover(DefaultImprover): + importer = GitLabAPIImporter + + +class IstioImprover(DefaultImprover): + importer = IstioImporter + + +class MozillaImprover(DefaultImprover): + importer = MozillaImporter + + +class NginxImprover(DefaultImprover): + importer = NginxImporter + + +class NpmImprover(DefaultImprover): + importer = NpmImporter + + +class OpensslImprover(DefaultImprover): + importer = OpensslImporter + + +class PostgreSQLImprover(DefaultImprover): + importer = PostgreSQLImporter + + +class ProjectKBMSRImprover(DefaultImprover): + importer = ProjectKBMSRImporter + + +class PyPaImprover(DefaultImprover): + importer = PyPaImporter + + +class PyPIImprover(DefaultImprover): + importer = PyPIImporter + + +class RedhatImprover(DefaultImprover): + importer = RedhatImporter + + +class RetireDotnetImprover(DefaultImprover): + importer = RetireDotnetImporter + + +class SUSESeverityScoreImprover(DefaultImprover): + importer = SUSESeverityScoreImporter + + +class UbuntuImprover(DefaultImprover): + importer = UbuntuImporter + + +class UbuntuUSNImprover(DefaultImprover): + importer = UbuntuUSNImporter + + +class XenImprover(DefaultImprover): + importer = XenImporter diff --git a/vulnerabilities/tests/test_importer_specific_improver.py b/vulnerabilities/tests/test_importer_specific_improver.py new file mode 100644 index 000000000..880703ec5 --- /dev/null +++ b/vulnerabilities/tests/test_importer_specific_improver.py @@ -0,0 +1,31 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import datetime + +import pytest + +from vulnerabilities.importers.nvd import NVDImporter +from vulnerabilities.improve_runner import ImproveRunner +from vulnerabilities.improvers.importer_specific_improver import NVDImprover +from vulnerabilities.models import Advisory +from vulnerabilities.models import Alias + + +@pytest.mark.django_db +def test_improvement_of_importer_specific_advisories(): + Advisory.objects.create( + aliases=["CVE-2021-22"], + summary="TEST", + created_by=NVDImporter.qualified_name, + date_collected=datetime.datetime.now(tz=datetime.timezone.utc), + ) + ImproveRunner(NVDImprover).run() + alias = Alias.objects.filter(alias="CVE-2021-22").first() + assert alias is not None diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index f51a1691b..d9c3f62fa 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -12,7 +12,7 @@ import warnings from pathlib import Path -__version__ = "33.3.0" +__version__ = "33.4.0" def command_line():