From 01e06fef0959c277f28c77714b48bb969f14648b Mon Sep 17 00:00:00 2001 From: jkaufman-logrhythm Date: Fri, 21 Jun 2024 12:05:06 -0600 Subject: [PATCH 1/3] Initial POC of LogRhythm Siem for search api --- README.md | 1 + .../platforms/logrhythm_siem/default.yml | 309 +++++++++ .../platforms/logrhythm_siem/__init__.py | 2 + .../platforms/logrhythm_siem/const.py | 94 +++ .../logrhythm_siem/escape_manager.py | 37 ++ .../platforms/logrhythm_siem/mapping.py | 47 ++ .../logrhythm_siem/renders/__init__.py | 0 .../renders/logrhythm_siem_query.py | 518 +++++++++++++++ .../renders/logrhythm_siem_rule.py | 610 ++++++++++++++++++ 9 files changed, 1618 insertions(+) create mode 100644 uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/const.py create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/renders/__init__.py create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py create mode 100644 uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py diff --git a/README.md b/README.md index dcc2be8a..f9133f6d 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Uncoder IO can be run on-prem without a need for an internet connection, thus su - FortiSIEM Rule - `fortisiem-rule` - LogRhythm Axon Rule - `axon-ads-rule` - LogRhythm Axon Query - `axon-ads-query` +- LogRhythm SIEM Query - `siem-json-query` IOC-based queries can be generated in the following formats: diff --git a/uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml b/uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml new file mode 100644 index 00000000..dd72d6c7 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/logrhythm_siem/default.yml @@ -0,0 +1,309 @@ +platform: LogRhythm SIEM +source: default + + +field_mapping: + EventID: vendor_information.id + Channel: general_information.log_source.type_name + ComputerName: origin.host.name + FileName: object.file.name + ProcessId: object.process.id + Image: object.process.name + AccountEmail: unattributed.account.email_address + ContextInfo: general_information.raw_message + CurrentDirectory: object.process.path + ParentProcessId: object.process.parent_process.id + ParentImage: object.process.parent_process.path + ParentCommandLine: object.process.parent_process.command_line + TargetFilename: object.file.name + SourceIp: origin.host.ip_address.value + SourceHostname: origin.host.name + SourcePort: origin.host.network_port.value + DestinationIp: target.host.ip_address.value + DestinationHostname: + - target.host.name + - target.host.domain + DestinationPort: target.host.network_port.value + DestinationPortName: action.network.protocol.name + ImageLoaded: object.file.path + SignatureStatus: object.process.signature.status + SourceProcessId: object.process.id + SourceImage: object.process.name + Device: object.process.path + Destination: object.process.name + QueryName: action.dns.query + QueryStatus: action.dns.result + CommandName: object.process.command_line + CommandPath: object.process.path + HostApplication: object.script.command_line + HostName: origin.host.name + ScriptName: object.script.name + ScriptBlockText: object.script.command_line + ScriptBlockId: object.script.id + Application: object.process.name + ClientAddress: origin.host.ip_address.value + ClientName: origin.host.domain.name + DestAddress: target.host.ip_address.value + DestPort: target.host.network_port.value + IpAddress: origin.host.ip_address.value + IpPort: origin.host.network_port.value + NewProcessId: object.process.id + NewProcessName: object.process.name + ParentProcessName: object.process.parent_process.name + ProcessName: object.process.name + SourceAddress: origin.host.ip_address.value + WorkstationName: origin.host.name + destination.port: target.host.network_port.value + dst: target.host.ip_address.value + dst_ip: target.host.ip_address.value + dst_port: target.host.network_port.value + network_application: + - action.network.protocol.name + - object.url.protocol + network_protocol: action.network.protocol.name + proto: action.network.protocol.name + src: origin.host.ip_address.value + src_ip: origin.host.ip_address.value + src_port: origin.host.network_port.value + action: action.command + mqtt_action: action.command + smb_action: action.command + tunnel_action: action.command + arg: object.process.command_args + ftp_arg: object.process.command_args + mysql_arg: object.process.command_args + pop3_arg: object.process.command_args + client: origin.host.ip_address.value + command: action.command + ftp_command: action.command + irc_command: action.command + pop3_command: action.command + duration: action.duration + from: origin.account.email_address + kerberos_from: origin.account.email_address + smtp_from: origin.account.email_address + method: action.network.http_method + http_method: action.network.http_method + sip_method: action.network.http_method + name: object.file.name + smb_files_name: object.file.name + software_name: object.file.name + weird_name: object.file.name + path: object.file.path + smb_mapping_path: object.file.path + smb_files_path: object.file.path + smtp_files_path: object.file.path + password: object.file.name + reply_to: target.account.email_address + response_body_len: action.network.byte_information.received + request_body_len: action.network.byte_information.sent + rtt: action.duration + status_code: action.result.code + known_certs_subject: object.certificate.subject + sip_subject: object.email_message.subject + smtp_subject: object.email_message.subject + ssl_subject: object.certificate.subject + username: origin.account.name + uri: object.url.path + user: origin.account.name + user_agent: action.user_agent + http_user_agent: action.user_agent + gquic_user_agent: action.user_agent + sip_user_agent: action.user_agent + smtp_user_agent: action.user_agent + version: object.file.version + gquic_version: object.file.version + http_version: object.file.version + ntp_version: object.file.version + socks_version: object.file.version + snmp_version: object.file.version + ssh_version: object.file.version + tls_version: object.file.version + answer: action.dns.result + question_length: action.network.byte_information.total + record_type: action.dns.record_type + parent_domain: target.host.domain + cs-bytes: action.network.byte_information.received + r-dns: target.host.domain + sc-bytes: action.network.byte_information.received + sc-status: action.result.code + c-uri: object.url.complete + c-uri-extension: object.url.type + c-uri-query: object.url.query + c-uri-stem: object.url.path + c-useragent: action.user_agent + cs-host: + - target.host.name + - target.host.domain + cs-method: action.network.http_method + cs-version: object.file.version + uid: action.session.id + endpoint: origin.host.name + domain: target.host.domain + host_name: target.host.name + client_fqdn: origin.host.name + requested_addr: target.host.ip_address.value + server_addr: target.host.ip_address.value + qtype: action.dns.record_type + qtype_name: action.dns.record_type + query: action.dns.query + rcode_name: action.dns.result + md5: unattributed.hash.md5 + sha1: unattributed.hash.sha1 + sha256: unattributed.hash.sha256 + sha512: unattributed.hash.sha512 + filename: object.file.name + host: + - unattributed.host.name + - unattributed.host.ip_address.value + domainname: unattributed.host.name + hostname: unattributed.host.name + server_nb_computer_name: unattributed.host.name + server_tree_name: unattributed.host.name + server_dns_computer_name: unattributed.host.name + machine: unattributed.host.name + os: origin.host.os.platform + mac: unattributed.host.mac_address + result: + - action.result.message + - action.result.code + - action.result.reason + mailfrom: origin.account.email_address + rcptto: target.account.email_address + second_received: target.account.email_address + server_name: unattributed.host.name + c-ip: origin.host.ip_address.value + cs-uri: object.url.path + cs-uri-query: object.url.query + cs-uri-stem: object.url.path + clientip: origin.host.ip_address.value + clientIP: origin.host.ip_address.value + dest_domain: + - target.host.name + - target.host.domain + dest_ip: target.host.ip_address.value + dest_port: target.host.network_port.value + agent.version: object.file.version + destination.hostname: + - target.host.name + - target.host.domain + DestinationAddress: + - target.host.name + - target.host.domain + - target.host.ip_address.value + DestinationIP: target.host.ip_address.value + dst-ip: target.host.ip_address.value + dstip: target.host.ip_address.value + dstport: target.host.ip_address.value + Host: target.host.name + HostVersion: object.file.version + http_host: + - target.host.name + - target.host.domain + - target.host.ip_address.value + http_uri: object.url.path + http_url: object.url.complete + http.request.url-query-params: object.url.query + HttpMethod: action.network.http_method + in_url: object.url.path + post_url_parameter: object.url.path + Request_Url: object.url.complete + request_url: object.url.complete + request_URL: object.url.complete + RequestUrl: object.url.complete + resource.url: object.url.path + resource.URL: object.url.path + sc_status: action.result.code + sender_domain: + - target.host.name + - target.host.domain + service.response_code: action.result.code + source: + - origin.host.name + - origin.host.domain.name + - origin.host.ip_address.value + SourceAddr: origin.host.ip_address.value + SourceIP: origin.host.ip_address.value + SourceNetworkAddress: origin.host.ip_address.value + srcip: origin.host.ip_address.value + Status: action.result.code + status: action.result.code + url: object.url.path + URL: object.url.path + url_query: object.url.query + url.query: object.url.query + uri_path: object.url.path + user_agent.name: action.user_agent + user-agent: action.user_agent + User-Agent: action.user_agent + useragent: action.user_agent + UserAgent: action.user_agent + User_Agent: action.user_agent + web_dest: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + web.dest: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + Web.dest: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + web.host: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + Web.host: + - target.host.name + - target.host.domain + - target.host.ip_address.value + - object.url.domain + web_method: action.network.http_method + Web_method: action.network.http_method + web.method: action.network.http_method + Web.method: action.network.http_method + web_src: origin.host.ip_address.value + web_status: action.result.code + Web_status: action.result.code + web.status: action.result.code + Web.status: action.result.code + web_uri: object.url.path + web_url: object.url.complete + destination.ip: target.host.ip_address.value + source.ip: origin.host.ip_address.value + source.port: origin.host.ip_address.value + Computer: + - target.host.name + - target.host.domain + - target.host.ip_address.value + OriginalFileName: object.file.name + User: origin.account.name + EventType: action.command + TargetObject: + - object.registry_object.key + - object.registry_object.path + - object.resource.name + CommandLine: object.process.command_line + type: + - action.command + - action.type + - action.session.type + a0: + - object.process.command_line + - object.process.command_args + - object.process.name + cs-user-agent: action.user_agent + blocked: + - action.message + - action.result.reason + cs-ip: origin.host.ip_address.value + SubjectLogonId: action.session.id + SubjectUserName: origin.account.name + SubjectUserSid: origin.account.id + SubjectDomainName: origin.account.domain diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py b/uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py new file mode 100644 index 00000000..8a1fbde6 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/__init__.py @@ -0,0 +1,2 @@ +# from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_query import LogRhythmSiemQueryRender # noqa: F401 +from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_rule import LogRhythmSiemRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/const.py b/uncoder-core/app/translator/platforms/logrhythm_siem/const.py new file mode 100644 index 00000000..2e8902ef --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/const.py @@ -0,0 +1,94 @@ +from app.translator.core.custom_types.meta_info import SeverityType +from app.translator.core.models.platform_details import PlatformDetails + +UNMAPPED_FIELD_DEFAULT_NAME = "general_information.raw_message" + +DEFAULT_LOGRHYTHM_Siem_RULE = { + "title": "Default LogRhythm Siem rule", + "version": 1, + "description": "Default LogRhythm Siem rule description.", + "maxMsgsToQuery": 30000, + "logCacheSize": 10000, + "aggregateLogCacheSize": 10000, + "queryTimeout": 60, + "isOriginatedFromWeb": False, + "webLayoutId": 0, + "queryRawLog": True, + "queryFilter": { + "msgFilterType": 2, + "isSavedFilter": False, + "filterGroup": { + "filterItemType": 1, + "fieldOperator": 1, + "filterMode": 1, + "filterGroupOperator": 0, + + "filterItems":"query", + "name": "Filter Group", + # "raw": "query" # FOR DEBUG REASONS + } + }, + "queryEventManager": False, + "useDefaultLogRepositories": True, + "dateCreated": "2024-06-05T22:47:06.3683942Z", + "dateSaved": "2024-06-05T22:47:06.3683942Z", + "dateUsed": "2024-06-05T22:47:06Z", + "includeDiagnosticEvents": True, + "searchMode": 2, + "webResultMode": 0, + "nextPageToken": "", + "pagedTimeout": 300, + "restrictedUserId": 0, + "createdVia": 0, + "searchType": 1, + "queryOrigin": 0, + "searchServerIPAddress": None, + "dateCriteria": { + "useInsertedDate": False, + "lastIntervalValue": 24, + "lastIntervalUnit": 7 + }, + "repositoryPattern": "", + "ownerId": 227, + "searchId": 0, + "queryLogSourceLists": [], + "queryLogSources": [], + "logRepositoryIds": [], + "refreshRate": 0, + "isRealTime": False, + "objectSecurity": { + "objectId": 0, + "objectType": 20, + "readPermissions": 0, + "writePermissions": 0, + "entityId": 1, + "ownerId": 227, + "canEdit": True, + "canDelete": False, + "canDeleteObject": False, + "entityName": "", + "ownerName": "", + "isSystemObject": True + }, + "enableIntelligentIndexing": False +} + +PLATFORM_DETAILS = {"group_id": "siem-ads", "group_name": "LogRhythm Siem"} + +LOGRHYTHM_Siem_QUERY_DETAILS = { + "platform_id": "siem-ads-query", + "name": "LogRhythm Siem Query", + "platform_name": "Query", + **PLATFORM_DETAILS, +} + +LOGRHYTHM_Siem_RULE_DETAILS = { + "platform_id": "siem-ads-rule", + "name": "LogRhythm Siem Search API", + "platform_name": "Search API", + "first_choice": 0, + **PLATFORM_DETAILS, +} + +# logrhythm_siem_query_details = PlatformDetails(**LOGRHYTHM_Siem_QUERY_DETAILS) +logrhythm_siem_rule_details = PlatformDetails(**LOGRHYTHM_Siem_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py b/uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py new file mode 100644 index 00000000..ec576ea6 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/escape_manager.py @@ -0,0 +1,37 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class LogRhythmQueryEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"'", escape_symbols=r"''")], + ValueType.regex_value: [ + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), + EscapeDetails(pattern=r"\*", escape_symbols=r"\\*"), + EscapeDetails(pattern=r"\.", escape_symbols=r"\\."), + EscapeDetails(pattern=r"\^", escape_symbols=r"\\^"), + EscapeDetails(pattern=r"\$", escape_symbols=r"\\$"), + EscapeDetails(pattern=r"\|", escape_symbols=r"\\|"), + EscapeDetails(pattern=r"\?", escape_symbols=r"\\?"), + EscapeDetails(pattern=r"\+", escape_symbols=r"\\+"), + EscapeDetails(pattern=r"\(", escape_symbols=r"\\("), + EscapeDetails(pattern=r"\)", escape_symbols=r"\\)"), + EscapeDetails(pattern=r"\[", escape_symbols=r"\\["), + EscapeDetails(pattern=r"\]", escape_symbols=r"\\]"), + EscapeDetails(pattern=r"\{", escape_symbols=r"\\{"), + EscapeDetails(pattern=r"\}", escape_symbols=r"\\}"), + ], + } + + +class LogRhythmRuleEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"'", escape_symbols=r"''")] + } + + +logrhythm_query_escape_manager = LogRhythmQueryEscapeManager() +logrhythm_rule_escape_manager = LogRhythmRuleEscapeManager() diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py b/uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py new file mode 100644 index 00000000..8e032a08 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/mapping.py @@ -0,0 +1,47 @@ +from typing import Optional + +from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping + + +class LogRhythmSiemLogSourceSignature(LogSourceSignature): + def __init__(self, default_source: Optional[dict] = None): + self._default_source = default_source or {} + + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "general_information.log_source.type_name" + + +class LogRhythmSiemMappings(BasePlatformMappings): + def prepare_mapping(self) -> dict[str, SourceMapping]: + source_mappings = {} + for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) + fields_mapping = self.prepare_fields_mapping(field_mapping=mapping_dict.get("field_mapping", {})) + source_mappings[DEFAULT_MAPPING_NAME] = SourceMapping( + source_id=DEFAULT_MAPPING_NAME, log_source_signature=log_source_signature, fields_mapping=fields_mapping + ) + return source_mappings + + def prepare_log_source_signature(self, mapping: dict) -> LogRhythmSiemLogSourceSignature: + default_log_source = mapping.get("default_log_source") + return LogRhythmSiemLogSourceSignature(default_source=default_log_source) + + def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMapping]: + suitable_source_mappings = [] + for source_mapping in self._source_mappings.values(): + if source_mapping.source_id == DEFAULT_MAPPING_NAME: + continue + + if source_mapping.fields_mapping.is_suitable(field_names): + suitable_source_mappings.append(source_mapping) + + if not suitable_source_mappings: + suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] + + return suitable_source_mappings + + +logrhythm_siem_mappings = LogRhythmSiemMappings(platform_dir="logrhythm_siem") diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/__init__.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py new file mode 100644 index 00000000..eee166b3 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py @@ -0,0 +1,518 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +import json +from typing import Union + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.custom_types.tokens import LogicalOperatorType +from app.translator.core.custom_types.values import ValueType +from app.translator.core.exceptions.core import StrictPlatformException +from app.translator.core.exceptions.render import BaseRenderException +from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.identifier import Identifier +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.managers import render_manager +# from app.translator.platforms.logrhythm_siem.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_siem_query_details +from app.translator.platforms.logrhythm_siem.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_siem_rule_details +from app.translator.platforms.logrhythm_siem.escape_manager import logrhythm_query_escape_manager +from app.translator.platforms.logrhythm_siem.mapping import LogRhythmSiemMappings, logrhythm_siem_mappings + + +class LogRhythmRegexRenderException(BaseRenderException): + ... + + +class LogRhythmSiemFieldValue(BaseQueryFieldValue): + # details: PlatformDetails = logrhythm_siem_query_details + details: PlatformDetails = logrhythm_siem_rule_details + escape_manager = logrhythm_query_escape_manager + + def __is_complex_regex(self, regex: str) -> bool: + regex_items = ("[", "]", "(", ")", "{", "}", "+", "?", "^", "$", "\\d", "\\w", "\\s", "-") + return any(v in regex for v in regex_items) + + def __is_contain_regex_items(self, value: str) -> bool: + regex_items = ("[", "]", "(", ")", "{", "}", "*", "+", "?", "^", "$", "|", ".", "\\d", "\\w", "\\s", "\\", "-") + return any(v in value for v in regex_items) + + def __regex_to_str_list(self, value: Union[int, str]) -> list[list[str]]: # noqa: PLR0912 + value_groups = [] + + stack = [] # [(element: str, escaped: bool)] + + for char in value: + if char == "\\": + if stack and stack[-1][0] == "\\" and stack[-1][1] is False: + stack.pop() + stack.append((char, True)) + else: + stack.append(("\\", False)) + elif char == "|": + if stack and stack[-1][0] == "\\" and stack[-1][1] is False: + stack.pop() + stack.append((char, True)) + elif stack: + value_groups.append("".join(element[0] for element in stack)) + stack = [] + else: + stack.append((char, False)) + if stack: + value_groups.append("".join(element[0] for element in stack if element[0] != "\\" or element[-1] is True)) + + joined_components = [] + for value_group in value_groups: + inner_joined_components = [] + not_joined_components = [] + for i in range(len(value_group)): + if value_group[i] == "*" and i > 0 and value_group[i - 1] != "\\": + inner_joined_components.append("".join(not_joined_components)) + not_joined_components = [] + else: + not_joined_components.append(value_group[i]) + if not_joined_components: + inner_joined_components.append("".join(not_joined_components)) + joined_components.append(inner_joined_components) + + return joined_components + + def __unmapped_regex_field_to_contains_string(self, field: str, value: str) -> str: + if self.__is_complex_regex(value): + raise LogRhythmRegexRenderException + values = self.__regex_to_str_list(value) + return ( + "(" + + self.or_token.join( + " AND ".join(f'{field} CONTAINS "{self.apply_value(value)}"' for value in value_list) + for value_list in values + ) + + ")" + ) + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, str): + return f'{field} = "{self.apply_value(value)}"' + if isinstance(value, list): + prepared_values = ", ".join(f"{self.apply_value(v)}" for v in value) + operator = "IN" if all(isinstance(v, str) for v in value) else "in" + return f"{field} {operator} [{prepared_values}]" + return f'{field} = "{self.apply_value(value)}"' + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} < {value}" + return f"{field} < '{self.apply_value(value)}'" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} <= {value}" + return f"{field} <= {self.apply_value(value)}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} > {value}" + return f"{field} > {self.apply_value(value)}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, int): + return f"{field} >= {value}" + return f"{field} >= {self.apply_value(value)}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + if isinstance(value, int): + return f"{field} != {value}" + return f"{field} != {self.apply_value(value)}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE): + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + return f'{field} CONTAINS "{self.apply_value(value)}"' + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + if isinstance(value, str) and field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + applied_value = self.apply_value(value, value_type=ValueType.regex_value) + return f'{field} matches ".*{applied_value}$"' + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + if isinstance(value, str) and field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + applied_value = self.apply_value(value, value_type=ValueType.regex_value) + return f'{field} matches "^{applied_value}.*"' + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field == UNMAPPED_FIELD_DEFAULT_NAME and self.__is_contain_regex_items(value): + if isinstance(value, str): + return self.__unmapped_regex_field_to_contains_string(field, value) + if isinstance(value, list): + return self.or_token.join( + self.__unmapped_regex_field_to_contains_string(field=field, value=v) for v in value + ) + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + if isinstance(value, str) and field == UNMAPPED_FIELD_DEFAULT_NAME: + return self.contains_modifier(field, value) + return f'{field} matches "{value}"' + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + if isinstance(value, list): + rendered_keywords = [f'{UNMAPPED_FIELD_DEFAULT_NAME} CONTAINS "{v}"' for v in value] + return f"({self.or_token.join(rendered_keywords)})" + return f'{UNMAPPED_FIELD_DEFAULT_NAME} CONTAINS "{value}"' + + # lrTODO: proper place for these, time constraints cheating and using them in siem_rules + # def generate_filter(self, field: str, field_type: str, val) -> str: + # ''' + # Most API returns under filter type in their own json field as shown below individually + # ''' + # print(f'Field passing>{field} ({field_type}) Val {type(val)}= {val}') + # f_n = self.field_translation(field) + # f_t = self.pull_filter_item(f_n, field_type) + # filter_structure = { + # "filterItemType": 0, + # "fieldOperator": 0, + # "filterMode": 2, + # "filterType": f_t[0], + # "values": [ + # { + # "filterType": f_t[0], + # "valueType": f_t[1], + # "value": val, + # "displayValue": "blank" + # } + # ], + # "name": f_n + # } + # return filter_structure + # # return json.dumps(filter_structure, indent=4) + + # def field_translation(self, field): + # if field == 'origin.account.name': + # field = 'User (Origin)' + # elif field == 'general_information.raw_message': + # field = 'Message' + # elif field in { 'object.process.command_line', + # 'object.script.command_line' + # }: + # field = 'Command' + # elif field in { 'object.registry_object.path', + # 'object.registry_object.key', + # 'object.resource.name' + # }: + # field = 'Object' + # elif field in { 'target.host.ip_address.value', + # 'target.host.ip_address.value' + # }: + # field = 'Address' + # elif field in { 'target.host.name', + # 'target.host.domain' + # }: + # field = 'DHost' + # elif field in { 'action.network.byte_information.received', + # 'action.network.byte_information.received' + # }: + # field = 'BytesIn' + # elif field == 'unattributed.host.mac_address': + # field = 'MAC' + # elif field == 'action.network.http_method': + # field = 'SIP' + # elif field in { 'origin.url.path', + # 'action.dns.query' + # }: + # field = 'URL' + # elif field == 'origin.host.domain': + # field = 'SHostName' + # elif field == 'target.host.domain': + # field = 'Host' + # elif field == 'action.network.byte_information.sent': + # field = 'BytesOut' + # elif field == 'action.network.byte_information.total': + # field = 'BytesInOut' + # elif field == 'object.process.name': + # field = 'Application' + # elif field == 'action.duration': + # field = 'Duration' + # elif field == 'process.parent_process.path': + # field = 'ParentProcessPath' + # elif field == 'object.process.parent_process.name': + # field = 'ParentProcessName' + # elif field == 'object.file.name' or field == 'TargetFilename': + # field = 'Object' + # elif field == 'target.host.network_port.value': + # field = 'Port' + # return field + + + # def pull_filter_item(self, f_type, f_v): + # if f_type == 'User (Origin)': + # f_type = 'Login' + + # filter_type = { + # 'IDMGroupForAccount': 53, + # 'Address': 44, + # 'Amount': 64, + # 'Application': 97, + # 'MsgClass': 10, + # 'Command': 112, + # 'CommonEvent': 11, + # 'Direction': 2, + # 'Duration': 62, + # 'Group': 38, + # 'BytesIn': 58, + # 'BytesOut': 59, + # 'BytesInOut': 95, + # 'DHost': 100, + # 'Host': 98, + # 'SHost': 99, + # 'ItemsIn': 60, + # 'ItemsOut': 61, + # 'ItemsInOut': 96, + # 'DHostName': 25, + # 'HostName': 23, + # 'SHostName': 24, + # 'KnownService': 16, + # 'DInterface': 108, + # 'Interface': 133, + # 'SInterface': 107, + # 'DIP': 19, + # 'IP': 17, + # 'SIP': 18, + # 'DIPRange': 22, + # 'IPRange': 20, + # 'SIPRange': 21, + # 'KnownDHost': 15, + # 'KnownHost': 13, + # 'KnownSHost': 14, + # 'Location': 87, + # 'SLocation': 85, + # 'DLocation': 86, + # 'MsgSource': 7, + # 'Entity': 6, + # 'RootEntity': 136, + # 'MsgSourceType': 9, + # 'DMAC': 104, + # 'MAC': 132, + # 'SMAC': 103, + # 'Message': 35, + # 'MPERule': 12, + # 'DNATIP': 106, + # 'NATIP': 126, + # 'SNATIP': 105, + # 'DNATIPRange': 125, + # 'NATIPRange': 127, + # 'SNATIPRange': 124, + # 'DNATPort': 115, + # 'NATPort': 130, + # 'SNATPort': 114, + # 'DNATPortRange': 129, + # 'NATPortRange': 131, + # 'SNATPortRange': 128, + # 'DNetwork': 50, + # 'Network': 51, + # 'SNetwork': 49, + # 'Object': 34, + # 'ObjectName': 113, + # 'Login': 29, + # 'IDMGroupForLogin': 52, + # 'Priority': 3, + # 'Process': 41, + # 'PID': 109, + # 'Protocol': 28, + # 'Quantity': 63, + # 'Rate': 65, + # 'Recipient': 32, + # 'Sender': 31, + # 'Session': 40, + # 'Severity': 110, + # 'Size': 66, + # 'Subject': 33, + # 'DPort': 27, + # 'Port': 45, + # 'SPort': 26, + # 'DPortRange': 47, + # 'PortRange': 48, + # 'SPortRange': 46, + # 'URL': 42, + # 'Account': 30, + # 'User': 43, + # 'IDMGroupForUser': 54, + # 'VendorMsgID': 37, + # 'Version': 111, + # 'SZone': 93, + # 'DZone': 94, + # 'FilterGroup': 1000, + # 'PolyListItem': 1001, + # 'Domain': 39, + # 'DomainOrigin': 137, + # 'Hash': 138, + # 'Policy': 139, + # 'VendorInfo': 140, + # 'Result': 141, + # 'ObjectType': 142, + # 'CVE': 143, + # 'UserAgent': 144, + # 'ParentProcessId': 145, + # 'ParentProcessName': 146, + # 'ParentProcessPath': 147, + # 'SerialNumber': 148, + # 'Reason': 149, + # 'Status': 150, + # 'ThreatId': 151, + # 'ThreatName': 152, + # 'SessionType': 153, + # 'Action': 154, + # 'ResponseCode': 155, + # 'UserOriginIdentityID': 167, + # 'Identity': 160, + # 'UserImpactedIdentityID': 168, + # 'SenderIdentityID': 169, + # 'RecipientIdentityID': 170 + # } + + # value_type = { + # 'Byte': 0, + # 'Int16': 1, + # 'Int32': 2, + # 'Int64': 3, + # 'String': 4, + # 'IPAddress': 5, + # 'IPAddressrange': 6, + # 'TimeOfDay': 7, + # 'DateRange': 8, + # 'PortRange': 9, + # 'Quantity': 10, + # 'ListReference': 11, + # 'ListSet': 12, + # 'Null': 13, + # 'INVALID': 99 + # } + # return_value = [] + # if f_type in filter_type: + # r_f = filter_type[f_type] + # else: + # print(f'filterType name reference was not found: {f_type}') + # r_f = 0000 + # if f_v in value_type: + # r_v = value_type[f_v] + # else: + # print(f'filterType name reference was not found: {f_v}') + # r_v = 13 + # return_value.append(r_f) + # return_value.append(r_v) + # # v_t = value_type[v_type] + # return return_value + + +@render_manager.register +class LogRhythmSiemQueryRender(PlatformQueryRender): + details: PlatformDetails = logrhythm_siem_rule_details + + or_token = "OR" + and_token = "AND" + not_token = "NOT" + + field_value_map = LogRhythmSiemFieldValue(or_token=or_token) + query_pattern = "{prefix} AND {query}" + + mappings: LogRhythmSiemMappings = logrhythm_siem_mappings + comment_symbol = "//" + is_single_line_comment = True + is_strict_mapping = True + + # def generate(self): + # print('moo') + + def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 + return str(log_source_signature) + + def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + if isinstance(token, FieldValue): + try: + mapped_fields = self.map_field(token.field, source_mapping) + except StrictPlatformException: + try: + return self.field_value_map.apply_field_value( + field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value + ) + except LogRhythmRegexRenderException as exc: + raise LogRhythmRegexRenderException( + f"Uncoder does not support complex regexp for unmapped field:" + f" {token.field.source_name} for LogRhythm Siem" + ) from exc + if len(mapped_fields) > 1: + return self.group_token % self.operator_map[LogicalOperatorType.OR].join( + [ + self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.field_value_map.apply_field_value( + field=mapped_fields[0], operator=token.operator, value=token.value + ) + + return super().apply_token(token, source_mapping) + + def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + queries_map = {} + source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) + + for source_mapping in source_mappings: + prefix = self.generate_prefix(source_mapping.log_source_signature) + if "product" in query_container.meta_info.parsed_logsources: + prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" + else: + prefix = f"{prefix} CONTAINS anything" + + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + finalized_query = self.finalize_query( + prefix=prefix, + query=result, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query + queries_map[source_mapping.source_id] = finalized_query + + return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py new file mode 100644 index 00000000..c201ed23 --- /dev/null +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py @@ -0,0 +1,610 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +import ast +import copy +from datetime import datetime, timedelta +import json +import random +import re +import traceback +from typing import Optional + +from app.translator.core.custom_types.meta_info import SeverityType +from app.translator.core.mapping import SourceMapping +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer +from app.translator.managers import render_manager +from app.translator.platforms.logrhythm_siem.const import DEFAULT_LOGRHYTHM_Siem_RULE, logrhythm_siem_rule_details +from app.translator.platforms.logrhythm_siem.escape_manager import logrhythm_rule_escape_manager +from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_query import ( + # LogRhythmSiemFieldValue + LogRhythmSiemFieldValue, + LogRhythmSiemQueryRender, +) +from app.translator.tools.utils import get_rule_description_str + +_AUTOGENERATED_TEMPLATE = "Autogenerated LogRhythm Siem Rule" +_SEVERITIES_MAP = { + SeverityType.critical: SeverityType.critical, + SeverityType.high: SeverityType.high, + SeverityType.medium: SeverityType.medium, + SeverityType.low: SeverityType.low, + SeverityType.informational: SeverityType.low, +} + + +class LogRhythmSiemRuleFieldValue(LogRhythmSiemFieldValue): + details: PlatformDetails = logrhythm_siem_rule_details + escape_manager = logrhythm_rule_escape_manager + + +@render_manager.register +class LogRhythmSiemRuleRender(LogRhythmSiemQueryRender): +# class LogRhythmSiemRuleRender(): + details: PlatformDetails = logrhythm_siem_rule_details + or_token = "or" + field_value_map = LogRhythmSiemRuleFieldValue(or_token=or_token) + + + # Function to generate ISO 8601 formatted date within the last 24 hours + def generate_iso_date(self): + now = datetime.utcnow() + past_date = now - timedelta(hours=random.randint(0, 23), minutes=random.randint(0, 59), seconds=random.randint(0, 59)) + return past_date.isoformat() + 'Z' + + + def generate_timestamps(self): + iso_date = self.generate_iso_date() + date_created = iso_date + date_saved = iso_date + date_used = iso_date + date_used = (date_used.split('.')[0]) + "Z" + return_obj = [] + + return_obj.append(date_created) + return_obj.append(date_saved) + return_obj.append(date_used) + return return_obj + + + def finalize_query( + self, + prefix: str, + query: str, + functions: str, + meta_info: Optional[MetaInfoContainer] = None, + source_mapping: Optional[SourceMapping] = None, + not_supported_functions: Optional[list] = None, + *args, # noqa: ARG002 + **kwargs, # noqa: ARG002 + ) -> str: + query = super().finalize_query(prefix=prefix, query=query, functions=functions) + rule = copy.deepcopy(DEFAULT_LOGRHYTHM_Siem_RULE) + ''' Parse out query - originally for "filter". + example how it should look like: + { + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": 29, # Must be altered + "values": [ + { + "filterType": 29, # Must be altered + "valueType": 4, + "value": { + "value": "moo", + "matchType": 0 + }, + "displayValue": "moo" + } + ], + "name": "User (Origin)" + } + + ANDs & CONTAINS needs to be broken out + ''' + # rule["queryFilter"]["filterGroup"]["raw"] = query # DEBUG + query = self.gen_filter_item(query) + # rule["observationPipeline"]["pattern"]["operations"][0]["logObserved"]["filter"] = query + rule["queryFilter"]["filterGroup"]["filterItems"] = query + + rule["title"] = meta_info.title or _AUTOGENERATED_TEMPLATE + rule["description"] = get_rule_description_str( + description=meta_info.description or rule["description"] or _AUTOGENERATED_TEMPLATE, + author=meta_info.author, + license_=meta_info.license, + ) + + # Set the time, default is last 24 hours + gen_time = self.generate_timestamps() + rule["dateCreated"] = gen_time[0] + rule["dateSaved"] = gen_time[1] + rule["dateUsed"] = gen_time[2] + + json_rule = json.dumps(rule, indent=4, sort_keys=False) + if not_supported_functions: + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return json_rule + rendered_not_supported + return json_rule + + + def pull_filter_item(self, f_type): + if f_type == 'User (Origin)': + f_type = 'Login' + + filter_type = { + 'IDMGroupForAccount': 53, + 'Address': 44, + 'Amount': 64, + 'Application': 97, + 'MsgClass': 10, + 'Command': 112, + 'CommonEvent': 11, + 'Direction': 2, + 'Duration': 62, + 'Group': 38, + 'BytesIn': 58, + 'BytesOut': 59, + 'BytesInOut': 95, + 'DHost': 100, + 'Host': 98, + 'SHost': 99, + 'ItemsIn': 60, + 'ItemsOut': 61, + 'ItemsInOut': 96, + 'DHostName': 25, + 'HostName': 23, + 'SHostName': 24, + 'KnownService': 16, + 'DInterface': 108, + 'Interface': 133, + 'SInterface': 107, + 'DIP': 19, + 'IP': 17, + 'SIP': 18, + 'DIPRange': 22, + 'IPRange': 20, + 'SIPRange': 21, + 'KnownDHost': 15, + 'KnownHost': 13, + 'KnownSHost': 14, + 'Location': 87, + 'SLocation': 85, + 'DLocation': 86, + 'MsgSource': 7, + 'Entity': 6, + 'RootEntity': 136, + 'MsgSourceType': 9, + 'DMAC': 104, + 'MAC': 132, + 'SMAC': 103, + 'Message': 35, + 'MPERule': 12, + 'DNATIP': 106, + 'NATIP': 126, + 'SNATIP': 105, + 'DNATIPRange': 125, + 'NATIPRange': 127, + 'SNATIPRange': 124, + 'DNATPort': 115, + 'NATPort': 130, + 'SNATPort': 114, + 'DNATPortRange': 129, + 'NATPortRange': 131, + 'SNATPortRange': 128, + 'DNetwork': 50, + 'Network': 51, + 'SNetwork': 49, + 'Object': 34, + 'ObjectName': 113, + 'Login': 29, + 'IDMGroupForLogin': 52, + 'Priority': 3, + 'Process': 41, + 'PID': 109, + 'Protocol': 28, + 'Quantity': 63, + 'Rate': 65, + 'Recipient': 32, + 'Sender': 31, + 'Session': 40, + 'Severity': 110, + 'Size': 66, + 'Subject': 33, + 'DPort': 27, + 'Port': 45, + 'SPort': 26, + 'DPortRange': 47, + 'PortRange': 48, + 'SPortRange': 46, + 'URL': 42, + 'Account': 30, + 'User': 43, + 'IDMGroupForUser': 54, + 'VendorMsgID': 37, + 'Version': 111, + 'SZone': 93, + 'DZone': 94, + 'FilterGroup': 1000, + 'PolyListItem': 1001, + 'Domain': 39, + 'DomainOrigin': 137, + 'Hash': 138, + 'Policy': 139, + 'VendorInfo': 140, + 'Result': 141, + 'ObjectType': 142, + 'CVE': 143, + 'UserAgent': 144, + 'ParentProcessId': 145, + 'ParentProcessName': 146, + 'ParentProcessPath': 147, + 'SerialNumber': 148, + 'Reason': 149, + 'Status': 150, + 'ThreatId': 151, + 'ThreatName': 152, + 'SessionType': 153, + 'Action': 154, + 'ResponseCode': 155, + 'UserOriginIdentityID': 167, + 'Identity': 160, + 'UserImpactedIdentityID': 168, + 'SenderIdentityID': 169, + 'RecipientIdentityID': 170 + } + + value_type = { + 'Byte': 0, + 'Int16': 1, + 'Int32': 2, + 'Int64': 3, + 'String': 4, + 'IPAddress': 5, + 'IPAddressrange': 6, + 'TimeOfDay': 7, + 'DateRange': 8, + 'PortRange': 9, + 'Quantity': 10, + 'ListReference': 11, + 'ListSet': 12, + 'Null': 13, + 'INVALID': 99 + } + if f_type in filter_type: + r_f = filter_type[f_type] + else: + print(f'filterType name reference was not found: {f_type}') + r_f = 0000 + # v_t = value_type[v_type] + return r_f + + + def process_sub_conditions(self, sub_conditions, items): + parsed_conditions = [] + for sub_condition in sub_conditions: + try: + # Parse the sub_condition + # This generally is preventing many things from parsing properly + if sub_condition.startswith('target.host.network_port.value') or \ + sub_condition.startswith('general_information.log_source.type_name CONTAINS target.host.network_port.value'): + sub_condition = sub_condition.replace('general_information.log_source.type_name CONTAINS ','') + matches = re.findall(r'(target\.host\.network_port\.value) in \[([^\]]+)\]', sub_condition) + if matches: + for match in matches: + # lrTODO : this needs to go into process_match + # items = self.process_match(match) + field, value = match + f_t = self.field_translation(field) + port_list = [int(x.strip()) for x in value.split(',')] + for port in port_list: + items.append({ + "filterType": self.pull_filter_item(f_t), + "valueType": 4, + "value": { + "value": port, + "matchType": 0 + }, + "displayValue": f_t + }) + elif 'CONTAINS' in sub_condition: + print(f'CONTAINS sub_condition: {sub_condition}') + try: + matches = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition)[0] + except: + matches = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition) + if not matches: + matches = re.findall(r'(\w+\.\w+) CONTAINS "([^"]+)"', sub_condition) + if 'matches ' in sub_condition: + match = matches[0] + # Some instances have a matches after a contains, + s_match = sub_condition.split('matches ')[1] + s_c_filtered = s_match.replace('"','').replace('.*','') + result = (match, s_c_filtered) + # result = (match, match + s_c_filtered) + f_t = self.field_translation(result[0]) + v_t = result[1] + if result[0] in result[1]: + v_t = v_t.replace(result[0],'') + items.append({ + "filterType": self.pull_filter_item(v_t), + "valueType": 4, + "value": { + "value": result[1].replace('\\', '/'), + "matchType": 0 + }, + "displayValue": f_t + }) + if len(matches) == 2: + f_t = self.field_translation(matches[0]) + items.append({ + "filterType": self.pull_filter_item(f_t), + "valueType": 4, + "value": { + "value": matches[1].replace('\\', '/'), + "matchType": 0 + }, + "displayValue": f_t + }) + else: + for match in matches: + items = self.process_match(match) + + # Parse the sub_condition + elif 'AND' in sub_condition: + print(f'AND sub_condition: {sub_condition} length={len(sub_condition)}') + matches = re.findall(r'(\w+\.\w+\.\w+) AND "([^"]+)"', sub_condition) + if not matches: + matches = re.findall(r'(\w+\.\w+\.\w+) matches ([^\s]+)', sub_condition) + # field, value = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition)[0] + if matches: + if len(matches) == 2: + f_t = self.field_translation(matches[0][0]) + f_v = matches[0][1] + f_v = f_v.replace('\"','') + items.append({ + "filterType": self.pull_filter_item(f_t), + "valueType": 4, + "value": { + "value": f_v.replace('\\', '/'), + "matchType": 0 + }, + "displayValue": f_t + }) + else: + for match in matches: + items = self.process_match(match) + elif 'matches' in sub_condition: + matches = re.findall(r'(\w+\.\w+\.\w+) matches "([^"]+)"', sub_condition) + if matches: + for match in matches: + items = self.process_match(match) + elif 'origin.account.name' in sub_condition or 'User' in sub_condition: + if 'User' in sub_condition: + matches = re.findall(r'User: "([^"]+)"', sub_condition) + else: + matches = re.findall(r'origin\.account\.name = "([^"]+)"', sub_condition) + if matches: + print(f'origin.account.name -> {matches}') + for match in matches: + items = self.process_match(match) + elif 'object.file.name' in sub_condition or 'TargetFilename' in sub_condition: + if 'TargetFilename' in sub_condition: + matches = re.findall(r'TargetFilename: "([^"]+)"', sub_condition) + else: + matches = re.findall(r'object\.file\.name = "([^"]+)"', sub_condition) + if matches: + for match in matches: + + items = self.process_match(match) + except Exception as e: + print(f'Error processing sub_condition: {sub_condition}') + print(f'Error: {e}') + traceback.print_exc() + print(f'items END: {items}\nParsed_condition: {parsed_conditions}') + return items + + + + def gen_filter_item(self, query): + ''' Collection of general observations found that breaks translation ''' + # Removing the product[0] AND to be more global friendly with parsing + query = query.replace("\\\'product\\\'", "'product'") + query = query.replace('query_container.meta_info.parsed_logsources[\'product\'][0] AND ', '') + + # Random + query = query.replace('anything AND ', '') # LR Siem cannot really handle this query + + # Remove outer parentheses + # Split the query into individual conditions + query = re.sub(r'^\(\(|\)\)$', '', query) + print(f'Query Cleaned: {query}') + conditions = re.split(r'\)\s*or \s*\(', query) + parsed_conditions = [] + + for condition in conditions: + # Remove inner parentheses + condition = condition.strip('()') + condition = condition.replace('((', '') + condition = condition.replace('(', '') + sub_conditions = re.split(r'\s*or \s*', condition) + # sub_conditions = re.split(r'\s*AND\s*', condition) + items = [] + + current_found = parsed_conditions.append(self.process_sub_conditions(sub_conditions, items)) + parsed_conditions.append(current_found) + return parsed_conditions[0] + + + def field_translation(self, field): + if field == 'origin.account.name': + field = 'User (Origin)' + elif field == 'general_information.raw_message': + field = 'Message' + elif field in { 'object.process.command_line', + 'object.script.command_line' + }: + field = 'Command' + elif field in { 'object.registry_object.path', + 'object.registry_object.key', + 'object.resource.name' + }: + field = 'Object' + elif field in { 'target.host.ip_address.value', + 'target.host.ip_address.value' + }: + field = 'Address' + elif field in { 'target.host.name', + 'target.host.domain' + }: + field = 'DHost' + elif field in { 'action.network.byte_information.received', + 'action.network.byte_information.received' + }: + field = 'BytesIn' + elif field == 'unattributed.host.mac_address': + field = 'MAC' + elif field == 'action.network.http_method': + field = 'SIP' + elif field in { 'origin.url.path', + 'action.dns.query' + }: + field = 'URL' + elif field == 'origin.host.domain': + field = 'SHostName' + elif field == 'target.host.domain': + field = 'Host' + elif field == 'action.network.byte_information.sent': + field = 'BytesOut' + elif field == 'action.network.byte_information.total': + field = 'BytesInOut' + elif field == 'object.process.name': + field = 'Application' + elif field == 'action.duration': + field = 'Duration' + elif field == 'process.parent_process.path': + field = 'ParentProcessPath' + elif field == 'object.process.parent_process.name': + field = 'ParentProcessName' + elif field == 'object.file.name' or field == 'TargetFilename': + field = 'Object' + elif field == 'target.host.network_port.value': + field = 'Port' + return field + + + def process_match(self, match): + field, value = match + keywords = ["AND", "CONTAINS", "AND NOT", "IN", "NOT"] + items = [] + + # Regex to extract field and value + # pattern = re.compile(r'(\S+)\s+(CONTAINS|NOT |IN|NOT IN|AND|OR)\s+(.+)') + pattern = re.compile(r'((?:NOT \s+)?\S+)\s+(CONTAINS|in|IN|NOT IN|AND|OR|AND NOT|NOT)\s+(.+)') + + # Initial tuple block + i_field = field + i_block = value.split(' ')[0] + i_block = i_block.replace('"','') + i_v = (i_field, i_block) + # i_block = re.split(f'\\s+{keyword}\\s+', value) + # print(f'item append len : {len(i_v)}') + items.extend(self.item_append(i_v)) + # Split the value based on the keywords + for keyword in keywords: + if keyword in value: + parts = re.split(f'\\s+{keyword}\\s+', value) + for part in parts: + part = part.strip() + if part: + # MUST match array list AND, CONTAINS, etc. otherwise skip + match_obj = pattern.match(part) + if match_obj == None and part.startswith('NOT '): + k = part.replace('NOT ','') + match_obj = pattern.match(k) + if match_obj: + # Construct a new match tuple and process it + new_field = match_obj.group(1) + # lr_TODO : clean value + new_value = match_obj.group(3) + current_match = (new_field, self.clean_value(new_value)) + print(f'keyword current_match len: {len(current_match)}\nvalue: {current_match}') + items.extend(self.item_append(current_match)) + break + else: + # If no keywords are found, process the original match + no_keywords = (field, value) + print(f'else item append no keyword:{no_keywords}') + items.extend(self.item_append(no_keywords)) + + return items + + + def clean_value(self, value): + return_value = value.replace(')','') + return return_value + + + def item_append(self, match): + items = [] + field, value = match + f_t = self.field_translation(field=field) + if f_t == 'Port': + if isinstance(value, list) or value.startswith('[') and value.endswith(']'): + value = self.check_array(value) + port_list = [int(x.strip()) for x in value.split(',')] + for port in port_list: + items.append(self.new_item(f_t=f_t, value=port)) + else: + items.append(self.new_item(f_t=f_t, value=value)) + else: + items.append(self.new_item(f_t=f_t, value=value)) + return items + + + + def new_item(self, f_t, value): + # Clean if value is string, else just feed it through (could be port number) + if isinstance(value, str): + value = value.replace('\\', '/') + i = { + "filterType": self.pull_filter_item(f_t), + "valueType": 4, + "value": { + "value": value, + "matchType": 0 + }, + "displayValue": f_t + } + return i + + + def check_array(self, s): + if isinstance(s, str): + if re.match(r'^\[.*\]$', s.strip()): + try: + # Safely eval the str to a Py List + array = ast.literal_eval(s) + if isinstance(array, list): + return array + except (ValueError, SyntaxError): + pass + return None + else: + return s \ No newline at end of file From 22782696cd6d3f5d4df6e32e40cd6a8f802a50d2 Mon Sep 17 00:00:00 2001 From: jkaufman-logrhythm Date: Tue, 2 Jul 2024 12:34:29 -0600 Subject: [PATCH 2/3] Initial POC of LogRhythm Siem for search api - corrected --- .../renders/logrhythm_siem_rule.py | 295 +++++++++++------- 1 file changed, 183 insertions(+), 112 deletions(-) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py index c201ed23..9e5f3d2f 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py @@ -145,128 +145,129 @@ def finalize_query( def pull_filter_item(self, f_type): + print(f'f_type passing to pull_filter item: {f_type}') if f_type == 'User (Origin)': f_type = 'Login' filter_type = { - 'IDMGroupForAccount': 53, - 'Address': 44, - 'Amount': 64, - 'Application': 97, - 'MsgClass': 10, - 'Command': 112, - 'CommonEvent': 11, - 'Direction': 2, - 'Duration': 62, - 'Group': 38, + 'IDMGroupForAccount': [53, 2, 'User (Impacted) by Active Directory Group'], + 'Address': [44, 11, 'Address (Sender or Recipient)'], + 'Amount': [64, 10, 'Amount'], + 'Application': [97, 11, 'PolyList Item'], + 'MsgClass': [10, 2, 'Classification'], + 'Command': [112, 4, 'Command'], + 'CommonEvent': [11, 2, 'Common Event'], + 'Direction': [2, 2, 'Direction'], + 'Duration': [62, 10, 'Duration'], + 'Group': [38, 11, 'Group'], 'BytesIn': 58, 'BytesOut': 59, 'BytesInOut': 95, 'DHost': 100, - 'Host': 98, + 'Host': [98, 11, 'PolyList Item'], 'SHost': 99, 'ItemsIn': 60, 'ItemsOut': 61, - 'ItemsInOut': 96, + 'ItemsInOut': [96, 10, 'Host (Impacted) Packets Total'], 'DHostName': 25, - 'HostName': 23, + 'HostName': [23, 4, 'HostName (Origin or Impacted)'], 'SHostName': 24, - 'KnownService': 16, + 'KnownService': [16, 2, 'Known Application'], 'DInterface': 108, - 'Interface': 133, + 'Interface': [133, 4, 'Interface (Origin or Impacted)'], 'SInterface': 107, 'DIP': 19, - 'IP': 17, + 'IP': [17, 5, 'IP Address (Origin or Impacted)'], 'SIP': 18, 'DIPRange': 22, 'IPRange': 20, 'SIPRange': 21, 'KnownDHost': 15, - 'KnownHost': 13, + 'KnownHost': [13, 2, 'Known Host (Origin or Impacted)'], 'KnownSHost': 14, - 'Location': 87, - 'SLocation': 85, - 'DLocation': 86, - 'MsgSource': 7, - 'Entity': 6, + 'Location': [87, 2, 'Location (Origin or Impacted)'], + 'SLocation': [85, 2, 'Location (Origin)'], + 'DLocation': [86, 2, 'Location (Impacted)'], + 'MsgSource': [7, 2, 'Log Source'], + 'Entity': [6, 2, 'Log Source Entity'], 'RootEntity': 136, - 'MsgSourceType': 9, + 'MsgSourceType': [9, 2, 'Log Source Type'], 'DMAC': 104, - 'MAC': 132, + 'MAC': [132, 4, 'MAC Address (Origin or Impacted)'], 'SMAC': 103, - 'Message': 35, + 'Message': [35, 4, 'Log Message'], 'MPERule': 12, 'DNATIP': 106, - 'NATIP': 126, + 'NATIP': [126, 5, 'NAT IP Address (Origin or Impacted)'], 'SNATIP': 105, 'DNATIPRange': 125, 'NATIPRange': 127, 'SNATIPRange': 124, 'DNATPort': 115, - 'NATPort': 130, + 'NATPort': [130, 2, 'NAT TCP/UDP Port (Origin or Impacted)'], 'SNATPort': 114, 'DNATPortRange': 129, 'NATPortRange': 131, 'SNATPortRange': 128, 'DNetwork': 50, - 'Network': 51, + 'Network': [51, 2, 'Network (Origin or Impacted)'], 'SNetwork': 49, - 'Object': 34, - 'ObjectName': 113, + 'Object': [34, 11, 'Object'], + 'ObjectName': [113, 4, 'Object Name'], 'Login': 29, - 'IDMGroupForLogin': 52, - 'Priority': 3, - 'Process': 41, - 'PID': 109, - 'Protocol': 28, - 'Quantity': 63, - 'Rate': 65, - 'Recipient': 32, - 'Sender': 31, - 'Session': 40, - 'Severity': 110, - 'Size': 66, - 'Subject': 33, + 'IDMGroupForLogin': [52, 2, 'User (Origin) by Active Directory Group'], + 'Priority': [3, 10, 'Risk Based Priority (RBP)'], + 'Process': [41, 4, 'Process Name'], + 'PID': [109, 2, 'Process ID'], + 'Protocol': [28, 1, 'Protocol'], + 'Quantity': [63, 10, 'Quantity'], + 'Rate': [65, 10, 'Rate'], + 'Recipient': [32, 11, 'Recipient'], + 'Sender': [31, 11, 'Sender'], + 'Session': [40, 4, 'Session'], + 'Severity': [110, 4, 'Severity'], + 'Size': [66, 10, 'Size'], + 'Subject': [33, 4, 'Subject'], 'DPort': 27, - 'Port': 45, + 'Port': [45,2, 'TCP/UDP Port (Origin or Impacted)'], 'SPort': 26, 'DPortRange': 47, - 'PortRange': 48, + 'PortRange': [48, 9, 'TCP/UDP Port Range (Origin or Impacted)'], 'SPortRange': 46, - 'URL': 42, + 'URL': [42, 4, 'URL'], 'Account': 30, - 'User': 43, - 'IDMGroupForUser': 54, - 'VendorMsgID': 37, - 'Version': 111, + 'User': [43, 4, 'User (Origin or Impacted)'], + 'IDMGroupForUser': [54, 2, 'User by Active Directory Group'], + 'VendorMsgID': [37, 4, 'Vendor Message ID'], + 'Version': [111, 4, 'Version'], 'SZone': 93, 'DZone': 94, 'FilterGroup': 1000, 'PolyListItem': 1001, - 'Domain': 39, - 'DomainOrigin': 137, - 'Hash': 138, - 'Policy': 139, - 'VendorInfo': 140, - 'Result': 141, - 'ObjectType': 142, - 'CVE': 143, - 'UserAgent': 144, - 'ParentProcessId': 145, - 'ParentProcessName': 146, - 'ParentProcessPath': 147, - 'SerialNumber': 148, - 'Reason': 149, - 'Status': 150, - 'ThreatId': 151, - 'ThreatName': 152, - 'SessionType': 153, - 'Action': 154, - 'ResponseCode': 155, + 'Domain': [39, 11, 'Domain Impacted'], + 'DomainOrigin': [137, 11, 'Domain Origin'], + 'Hash': [138, 4, 'Hash'], + 'Policy': [139, 4, 'Policy'], + 'VendorInfo': [140, 4, 'Vendor Info'], + 'Result': [141, 4, 'Result'], + 'ObjectType': [142, 4, 'Object Type'], + 'CVE': [143,11, 'CVE'], + 'UserAgent': [144, 4, 'User Agent'], + 'ParentProcessId': [145, 4, 'Parent Process ID'], + 'ParentProcessName': [146, 4, 'Parent Process Name'], + 'ParentProcessPath': [147, 4, 'Parent Process Path'], + 'SerialNumber': [148, 4, 'Serial Number'], + 'Reason': [149, 4, 'Reason'], + 'Status': [150, 4, 'Status'], + 'ThreatId': [151, 4, 'Threat ID'], + 'ThreatName': [152, 4, 'Threat Name'], + 'SessionType': [153, 4, 'Session Type'], + 'Action': [154,13, 'Action'], + 'ResponseCode': [155, 4, 'Response Code'], 'UserOriginIdentityID': 167, 'Identity': 160, - 'UserImpactedIdentityID': 168, - 'SenderIdentityID': 169, + 'UserImpactedIdentityID': [168,2], + 'SenderIdentityID': [169, 2, 'Sender Identity'], 'RecipientIdentityID': 170 } @@ -312,19 +313,45 @@ def process_sub_conditions(self, sub_conditions, items): # items = self.process_match(match) field, value = match f_t = self.field_translation(field) + f_number = self.pull_filter_item(f_t) + print(f'f_number: {f_number}') port_list = [int(x.strip()) for x in value.split(',')] + collected_values = [] for port in port_list: - items.append({ - "filterType": self.pull_filter_item(f_t), - "valueType": 4, - "value": { - "value": port, - "matchType": 0 - }, - "displayValue": f_t + collected_values.append({ + "filterType": f_number[0], + "valueType": f_number[1], + "value": port, + "displayValue": str(port) }) + items.append({ + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values": collected_values, + "name": f_number[2] + # "name": f_t + }) + # for port in port_list: + # items.append({ + # "filterItemType": 0, + # "fieldOperator": 0, + # "filterMode": 1, + # "filterType": f_number[0], + # "values":[{ + # "filterType": f_number[0], + # "valueType": f_number[1], + # "value": { + # "value":port, + # "matchType": 0 + # }, + # "displayValue": str(port) + # }], + # "name": f_number[2] + # # "name": f_t + # }) elif 'CONTAINS' in sub_condition: - print(f'CONTAINS sub_condition: {sub_condition}') try: matches = re.findall(r'(\w+\.\w+\.\w+) CONTAINS "([^"]+)"', sub_condition)[0] except: @@ -339,28 +366,45 @@ def process_sub_conditions(self, sub_conditions, items): result = (match, s_c_filtered) # result = (match, match + s_c_filtered) f_t = self.field_translation(result[0]) + f_number = self.pull_filter_item(f_t) v_t = result[1] if result[0] in result[1]: v_t = v_t.replace(result[0],'') items.append({ - "filterType": self.pull_filter_item(v_t), - "valueType": 4, - "value": { - "value": result[1].replace('\\', '/'), - "matchType": 0 - }, - "displayValue": f_t + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[{ + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value":result[1].replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(result[1].replace('\\', '/')) + }], + "name": f_number[2] + # "name": f_t }) if len(matches) == 2: f_t = self.field_translation(matches[0]) + f_number = self.pull_filter_item(f_t) items.append({ - "filterType": self.pull_filter_item(f_t), - "valueType": 4, - "value": { - "value": matches[1].replace('\\', '/'), - "matchType": 0 - }, - "displayValue": f_t + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[{ + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value":matches[1].replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(matches[1].replace('\\', '/')) + }], + "name": f_number[2] }) else: for match in matches: @@ -368,7 +412,6 @@ def process_sub_conditions(self, sub_conditions, items): # Parse the sub_condition elif 'AND' in sub_condition: - print(f'AND sub_condition: {sub_condition} length={len(sub_condition)}') matches = re.findall(r'(\w+\.\w+\.\w+) AND "([^"]+)"', sub_condition) if not matches: matches = re.findall(r'(\w+\.\w+\.\w+) matches ([^\s]+)', sub_condition) @@ -378,14 +421,22 @@ def process_sub_conditions(self, sub_conditions, items): f_t = self.field_translation(matches[0][0]) f_v = matches[0][1] f_v = f_v.replace('\"','') + f_number = self.pull_filter_item(f_t) items.append({ - "filterType": self.pull_filter_item(f_t), - "valueType": 4, - "value": { - "value": f_v.replace('\\', '/'), - "matchType": 0 - }, - "displayValue": f_t + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[{ + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value": f_v.replace('\\', '/'), + "matchType": 0 + }, + "displayValue": str(f_v.replace('\\', '/')) + }], + "name": f_number[2] }) else: for match in matches: @@ -411,7 +462,6 @@ def process_sub_conditions(self, sub_conditions, items): matches = re.findall(r'object\.file\.name = "([^"]+)"', sub_condition) if matches: for match in matches: - items = self.process_match(match) except Exception as e: print(f'Error processing sub_condition: {sub_condition}') @@ -453,6 +503,7 @@ def gen_filter_item(self, query): def field_translation(self, field): + print(f'Field translation submission: {field}') if field == 'origin.account.name': field = 'User (Origin)' elif field == 'general_information.raw_message': @@ -495,7 +546,7 @@ def field_translation(self, field): elif field == 'action.network.byte_information.total': field = 'BytesInOut' elif field == 'object.process.name': - field = 'Application' + field = 'Process' elif field == 'action.duration': field = 'Duration' elif field == 'process.parent_process.path': @@ -583,15 +634,35 @@ def new_item(self, f_t, value): # Clean if value is string, else just feed it through (could be port number) if isinstance(value, str): value = value.replace('\\', '/') + f_number = self.pull_filter_item(f_t) + # i = { + # "filterType": self.pull_filter_item(f_t), + # "valueType": 4, + # "value": { + # "value": value, + # "matchType": 0 + # }, + # "displayValue": f_t + # } + i = { - "filterType": self.pull_filter_item(f_t), - "valueType": 4, - "value": { - "value": value, + "filterItemType": 0, + "fieldOperator": 0, + "filterMode": 1, + "filterType": f_number[0], + "values":[ + { + "filterType": f_number[0], + "valueType": f_number[1], + "value": { + "value": value.replace('\\', '/'), "matchType": 0 - }, - "displayValue": f_t - } + }, + "displayValue": str(value.replace('\\', '/')) + } + ], + "name": f_number[2] + } return i From 8d555744a54b67f948b500f9f0305875bd7663e8 Mon Sep 17 00:00:00 2001 From: jkaufman-logrhythm Date: Wed, 3 Jul 2024 13:42:02 -0600 Subject: [PATCH 3/3] Latest Fork Sync patch --- .../renders/logrhythm_siem_query.py | 284 ++---------------- .../renders/logrhythm_siem_rule.py | 15 +- 2 files changed, 25 insertions(+), 274 deletions(-) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py index eee166b3..7d710f94 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_query.py @@ -13,10 +13,10 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. +limitations under the License.a ----------------------------------------------------------------- """ -import json + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -30,7 +30,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager # from app.translator.platforms.logrhythm_siem.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_siem_query_details from app.translator.platforms.logrhythm_siem.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_siem_rule_details @@ -42,9 +42,9 @@ class LogRhythmRegexRenderException(BaseRenderException): ... -class LogRhythmSiemFieldValue(BaseQueryFieldValue): - # details: PlatformDetails = logrhythm_siem_query_details +class LogRhythmSiemFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logrhythm_siem_rule_details + # details: PlatformDetails = logrhythm_siem_query_details escape_manager = logrhythm_query_escape_manager def __is_complex_regex(self, regex: str) -> bool: @@ -156,7 +156,7 @@ def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"{field} != {value}" return f"{field} != {self.apply_value(value)}" - def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE): + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" return f'{field} CONTAINS "{self.apply_value(value)}"' @@ -197,278 +197,37 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 return f"({self.or_token.join(rendered_keywords)})" return f'{UNMAPPED_FIELD_DEFAULT_NAME} CONTAINS "{value}"' - # lrTODO: proper place for these, time constraints cheating and using them in siem_rules - # def generate_filter(self, field: str, field_type: str, val) -> str: - # ''' - # Most API returns under filter type in their own json field as shown below individually - # ''' - # print(f'Field passing>{field} ({field_type}) Val {type(val)}= {val}') - # f_n = self.field_translation(field) - # f_t = self.pull_filter_item(f_n, field_type) - # filter_structure = { - # "filterItemType": 0, - # "fieldOperator": 0, - # "filterMode": 2, - # "filterType": f_t[0], - # "values": [ - # { - # "filterType": f_t[0], - # "valueType": f_t[1], - # "value": val, - # "displayValue": "blank" - # } - # ], - # "name": f_n - # } - # return filter_structure - # # return json.dumps(filter_structure, indent=4) - - # def field_translation(self, field): - # if field == 'origin.account.name': - # field = 'User (Origin)' - # elif field == 'general_information.raw_message': - # field = 'Message' - # elif field in { 'object.process.command_line', - # 'object.script.command_line' - # }: - # field = 'Command' - # elif field in { 'object.registry_object.path', - # 'object.registry_object.key', - # 'object.resource.name' - # }: - # field = 'Object' - # elif field in { 'target.host.ip_address.value', - # 'target.host.ip_address.value' - # }: - # field = 'Address' - # elif field in { 'target.host.name', - # 'target.host.domain' - # }: - # field = 'DHost' - # elif field in { 'action.network.byte_information.received', - # 'action.network.byte_information.received' - # }: - # field = 'BytesIn' - # elif field == 'unattributed.host.mac_address': - # field = 'MAC' - # elif field == 'action.network.http_method': - # field = 'SIP' - # elif field in { 'origin.url.path', - # 'action.dns.query' - # }: - # field = 'URL' - # elif field == 'origin.host.domain': - # field = 'SHostName' - # elif field == 'target.host.domain': - # field = 'Host' - # elif field == 'action.network.byte_information.sent': - # field = 'BytesOut' - # elif field == 'action.network.byte_information.total': - # field = 'BytesInOut' - # elif field == 'object.process.name': - # field = 'Application' - # elif field == 'action.duration': - # field = 'Duration' - # elif field == 'process.parent_process.path': - # field = 'ParentProcessPath' - # elif field == 'object.process.parent_process.name': - # field = 'ParentProcessName' - # elif field == 'object.file.name' or field == 'TargetFilename': - # field = 'Object' - # elif field == 'target.host.network_port.value': - # field = 'Port' - # return field - - - # def pull_filter_item(self, f_type, f_v): - # if f_type == 'User (Origin)': - # f_type = 'Login' - - # filter_type = { - # 'IDMGroupForAccount': 53, - # 'Address': 44, - # 'Amount': 64, - # 'Application': 97, - # 'MsgClass': 10, - # 'Command': 112, - # 'CommonEvent': 11, - # 'Direction': 2, - # 'Duration': 62, - # 'Group': 38, - # 'BytesIn': 58, - # 'BytesOut': 59, - # 'BytesInOut': 95, - # 'DHost': 100, - # 'Host': 98, - # 'SHost': 99, - # 'ItemsIn': 60, - # 'ItemsOut': 61, - # 'ItemsInOut': 96, - # 'DHostName': 25, - # 'HostName': 23, - # 'SHostName': 24, - # 'KnownService': 16, - # 'DInterface': 108, - # 'Interface': 133, - # 'SInterface': 107, - # 'DIP': 19, - # 'IP': 17, - # 'SIP': 18, - # 'DIPRange': 22, - # 'IPRange': 20, - # 'SIPRange': 21, - # 'KnownDHost': 15, - # 'KnownHost': 13, - # 'KnownSHost': 14, - # 'Location': 87, - # 'SLocation': 85, - # 'DLocation': 86, - # 'MsgSource': 7, - # 'Entity': 6, - # 'RootEntity': 136, - # 'MsgSourceType': 9, - # 'DMAC': 104, - # 'MAC': 132, - # 'SMAC': 103, - # 'Message': 35, - # 'MPERule': 12, - # 'DNATIP': 106, - # 'NATIP': 126, - # 'SNATIP': 105, - # 'DNATIPRange': 125, - # 'NATIPRange': 127, - # 'SNATIPRange': 124, - # 'DNATPort': 115, - # 'NATPort': 130, - # 'SNATPort': 114, - # 'DNATPortRange': 129, - # 'NATPortRange': 131, - # 'SNATPortRange': 128, - # 'DNetwork': 50, - # 'Network': 51, - # 'SNetwork': 49, - # 'Object': 34, - # 'ObjectName': 113, - # 'Login': 29, - # 'IDMGroupForLogin': 52, - # 'Priority': 3, - # 'Process': 41, - # 'PID': 109, - # 'Protocol': 28, - # 'Quantity': 63, - # 'Rate': 65, - # 'Recipient': 32, - # 'Sender': 31, - # 'Session': 40, - # 'Severity': 110, - # 'Size': 66, - # 'Subject': 33, - # 'DPort': 27, - # 'Port': 45, - # 'SPort': 26, - # 'DPortRange': 47, - # 'PortRange': 48, - # 'SPortRange': 46, - # 'URL': 42, - # 'Account': 30, - # 'User': 43, - # 'IDMGroupForUser': 54, - # 'VendorMsgID': 37, - # 'Version': 111, - # 'SZone': 93, - # 'DZone': 94, - # 'FilterGroup': 1000, - # 'PolyListItem': 1001, - # 'Domain': 39, - # 'DomainOrigin': 137, - # 'Hash': 138, - # 'Policy': 139, - # 'VendorInfo': 140, - # 'Result': 141, - # 'ObjectType': 142, - # 'CVE': 143, - # 'UserAgent': 144, - # 'ParentProcessId': 145, - # 'ParentProcessName': 146, - # 'ParentProcessPath': 147, - # 'SerialNumber': 148, - # 'Reason': 149, - # 'Status': 150, - # 'ThreatId': 151, - # 'ThreatName': 152, - # 'SessionType': 153, - # 'Action': 154, - # 'ResponseCode': 155, - # 'UserOriginIdentityID': 167, - # 'Identity': 160, - # 'UserImpactedIdentityID': 168, - # 'SenderIdentityID': 169, - # 'RecipientIdentityID': 170 - # } - - # value_type = { - # 'Byte': 0, - # 'Int16': 1, - # 'Int32': 2, - # 'Int64': 3, - # 'String': 4, - # 'IPAddress': 5, - # 'IPAddressrange': 6, - # 'TimeOfDay': 7, - # 'DateRange': 8, - # 'PortRange': 9, - # 'Quantity': 10, - # 'ListReference': 11, - # 'ListSet': 12, - # 'Null': 13, - # 'INVALID': 99 - # } - # return_value = [] - # if f_type in filter_type: - # r_f = filter_type[f_type] - # else: - # print(f'filterType name reference was not found: {f_type}') - # r_f = 0000 - # if f_v in value_type: - # r_v = value_type[f_v] - # else: - # print(f'filterType name reference was not found: {f_v}') - # r_v = 13 - # return_value.append(r_f) - # return_value.append(r_v) - # # v_t = value_type[v_type] - # return return_value - @render_manager.register class LogRhythmSiemQueryRender(PlatformQueryRender): details: PlatformDetails = logrhythm_siem_rule_details + # details: PlatformDetails = logrhythm_siem_query_details or_token = "OR" and_token = "AND" not_token = "NOT" - field_value_map = LogRhythmSiemFieldValue(or_token=or_token) - query_pattern = "{prefix} AND {query}" + field_value_render = LogRhythmSiemFieldValueRender(or_token=or_token) mappings: LogRhythmSiemMappings = logrhythm_siem_mappings comment_symbol = "//" is_single_line_comment = True is_strict_mapping = True - # def generate(self): - # print('moo') + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"AND {query}" if query else "" def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 return str(log_source_signature) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: - if isinstance(token, FieldValue): + if isinstance(token, FieldValue) and token.field: try: mapped_fields = self.map_field(token.field, source_mapping) except StrictPlatformException: try: - return self.field_value_map.apply_field_value( + return self.field_value_render.apply_field_value( field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value ) except LogRhythmRegexRenderException as exc: @@ -476,20 +235,17 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Siem" ) from exc - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) - for field in mapped_fields - ] - ) - return self.field_value_map.apply_field_value( - field=mapped_fields[0], operator=token.operator, value=token.value + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] ) + return self.group_token % joined if len(mapped_fields) > 1 else joined return super().apply_token(token, source_mapping) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) diff --git a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py index 9e5f3d2f..b68c08c8 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_siem/renders/logrhythm_siem_rule.py @@ -34,7 +34,8 @@ from app.translator.platforms.logrhythm_siem.escape_manager import logrhythm_rule_escape_manager from app.translator.platforms.logrhythm_siem.renders.logrhythm_siem_query import ( # LogRhythmSiemFieldValue - LogRhythmSiemFieldValue, + # LogRhythmSiemFieldValue, + LogRhythmSiemFieldValueRender, LogRhythmSiemQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -49,7 +50,8 @@ } -class LogRhythmSiemRuleFieldValue(LogRhythmSiemFieldValue): +class LogRhythmSiemRuleFieldValue(LogRhythmSiemFieldValueRender): +# class LogRhythmSiemRuleFieldValue(LogRhythmSiemFieldValue): details: PlatformDetails = logrhythm_siem_rule_details escape_manager = logrhythm_rule_escape_manager @@ -145,7 +147,6 @@ def finalize_query( def pull_filter_item(self, f_type): - print(f'f_type passing to pull_filter item: {f_type}') if f_type == 'User (Origin)': f_type = 'Login' @@ -314,7 +315,6 @@ def process_sub_conditions(self, sub_conditions, items): field, value = match f_t = self.field_translation(field) f_number = self.pull_filter_item(f_t) - print(f'f_number: {f_number}') port_list = [int(x.strip()) for x in value.split(',')] collected_values = [] for port in port_list: @@ -452,7 +452,6 @@ def process_sub_conditions(self, sub_conditions, items): else: matches = re.findall(r'origin\.account\.name = "([^"]+)"', sub_condition) if matches: - print(f'origin.account.name -> {matches}') for match in matches: items = self.process_match(match) elif 'object.file.name' in sub_condition or 'TargetFilename' in sub_condition: @@ -467,7 +466,7 @@ def process_sub_conditions(self, sub_conditions, items): print(f'Error processing sub_condition: {sub_condition}') print(f'Error: {e}') traceback.print_exc() - print(f'items END: {items}\nParsed_condition: {parsed_conditions}') + # print(f'items END: {items}\nParsed_condition: {parsed_conditions}') return items @@ -484,7 +483,6 @@ def gen_filter_item(self, query): # Remove outer parentheses # Split the query into individual conditions query = re.sub(r'^\(\(|\)\)$', '', query) - print(f'Query Cleaned: {query}') conditions = re.split(r'\)\s*or \s*\(', query) parsed_conditions = [] @@ -503,7 +501,6 @@ def gen_filter_item(self, query): def field_translation(self, field): - print(f'Field translation submission: {field}') if field == 'origin.account.name': field = 'User (Origin)' elif field == 'general_information.raw_message': @@ -595,13 +592,11 @@ def process_match(self, match): # lr_TODO : clean value new_value = match_obj.group(3) current_match = (new_field, self.clean_value(new_value)) - print(f'keyword current_match len: {len(current_match)}\nvalue: {current_match}') items.extend(self.item_append(current_match)) break else: # If no keywords are found, process the original match no_keywords = (field, value) - print(f'else item append no keyword:{no_keywords}') items.extend(self.item_append(no_keywords)) return items