diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml index 9a4d96783..1877e661b 100644 --- a/.github/workflows/debian-package.yml +++ b/.github/workflows/debian-package.yml @@ -33,7 +33,7 @@ jobs: run: bash .github/workflows/scripts/debian-package.sh ${{ matrix.codename }} - name: Test packages installation - run: sudo apt install ~/artifacts/*.deb + run: sudo apt-get update && DEBIAN_FRONTEND="noninteractive" sudo apt-get install ~/artifacts/*.deb - name: Upload artifact if: ${{ github.event_name == 'push' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a5ff9025..73058125a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,6 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Development ### Data Format - - Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`, `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following @@ -40,6 +39,8 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; ``` +- added `severity` field to help with triaging received events (PR#2575 by Kamil Mańkowski). + To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. ### Bots #### Collectors diff --git a/NEWS.md b/NEWS.md index 4c948919e..27e3b20ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -27,6 +27,7 @@ ALTER TABLE events ADD "product.name" text; ALTER TABLE events ADD "product.vendor" text; ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; +ALTER TABLE events ADD severity varchar(10); ``` ### Configuration diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index b9e57d6e2..4cca6c144 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -253,6 +253,12 @@ "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.", "type": "URL" }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source. Allowed values: critical (highly critical vulnerabilities being actively exploited and pose a very high likelihood of compromise. For example, RCEs, sensitive data access), high (end of life systems, accessible internal systems that should not be exposed, risk of data leaks, malware drone and sinkhole events), medium (DDoS-amplifiers, unencrypted services requiring login, vulnerabilities requiring MITM to exploit, attacks need prior knowledge), low (deviation from best practice, little to no practical way to exploit, but setup is not ideal), info (informational only, no known risk), undefined (unknown or undetermined)", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString" + }, "source.abuse_contact": { "description": "Abuse contact for source address. A comma separated list.", "type": "LowercaseString" diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 3da3e95fc..208684c03 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -42,8 +42,8 @@ 'v322_url_replacement', 'v322_removed_feeds_and_bots', 'v340_deprecations', - 'v341_blueliv_removal', - 'v342_new_fields' + 'v350_blueliv_removal', + 'v350_new_fields', ] @@ -976,7 +976,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization -def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): +def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ Remove blueliv collector and parser """ @@ -999,7 +999,7 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): return message, configuration, harmonization -def v342_new_fields(configuration, harmonization, dry_run, **kwargs): +def v350_new_fields(configuration, harmonization, dry_run, **kwargs): """ Add new fields to IntelMQ Data Format """ @@ -1011,6 +1011,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): resource_filename("intelmq", "etc/harmonization.conf") ) for field in [ + "severity", "product.full_name", "product.name", "product.vendor", @@ -1056,8 +1057,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), (v341_blueliv_removal, )), - ((3, 4, 2), (v342_new_fields, )), + ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 19faecb85..7e09d549d 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -57,6 +57,7 @@ CREATE TABLE events ( "raw" text, "rtir_id" integer, "screenshot_url" text, + "severity" varchar(10), "source.abuse_contact" text, "source.account" text, "source.allocated" timestamp with time zone, diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 6021a0fc0..047ecb4d4 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -616,7 +616,7 @@ "module": "intelmq.bots.collectors.twitter.collector", }, } -V341_BLUELIV_REMOVAL = { +V350_BLUELIV_REMOVAL = { "global": {}, "blueliv-collector": { "module": "intelmq.bots.collectors.blueliv.collector_crimeserver" @@ -865,16 +865,16 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) - def test_v341_blueliv_removal(self): - """ Test v341_blueliv_removal deprecation warning """ - result = upgrades.v341_blueliv_removal(V341_BLUELIV_REMOVAL, {}, False) + def test_v350_blueliv_removal(self): + """ Test v350_blueliv_removal deprecation warning """ + result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False) self.assertIn('blueliv-collector', result[0]) self.assertIn('blueliv-parser', result[0]) - self.assertEqual(V341_BLUELIV_REMOVAL, result[1]) + self.assertEqual(V350_BLUELIV_REMOVAL, result[1]) - def test_v342_new_fields(self): + def test_v350_new_fields(self): """ Test adding new harmonisation fields """ - result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) + result = upgrades.v350_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) @@ -882,6 +882,9 @@ def test_v342_new_fields(self): self.assertIn("product.vendor", result[2]["event"]) self.assertIn("product.version", result[2]["event"]) self.assertIn("product.vulnerabilities", result[2]["event"]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("severity", result[2]["event"]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,