Optionally add prefix to each log line to see the origin

.gitignore

@@ -0,0 +1 @@
+le-aws-cloudwatch-lambda.zip

Makefile

@@ -0,0 +1,11 @@
+.PHONY: certifi zip dist
+
+# get the latest certifi
+certifi:
+	pip install --upgrade --target .  certifi && rm -rf certifi-201*dist-info
+
+# package for upload
+zip:
+	zip le-aws-cloudwatch-lambda le_cloudwatch.py certifi/*
+
+dist: certifi zip

README.md

@@ -22,22 +22,25 @@
    * Set runtime to Python 2.7
 
 4. Upload function code:
-      * Create a .ZIP file, containing ```le_cloudwatch.py``` and the folder ```certifi```
-        * Make sure the files and ```certifi``` folder are in the **root** of the ZIP archive
-      * Choose "Upload a .ZIP file" in "Code entry type" dropdown and upload the archive created in previous step
+   * Optionally use `make certifi` to ensure upgraded to latest on pypi
+   * Create a zip file
+     * Run `make zip` to get the file
+     * _or_ create a .ZIP file, containing the updated ```le_cloudwatch.py``` and the folder ```certifi```
+       * Make sure the files and ```certifi``` folder are in the **root** of the ZIP archive
+   * Choose "Upload a .ZIP file" in "Code entry type" dropdown and upload the archive created in previous step
 
 5. Lambda function handler and role
    * Change the "Handler" value to ```le_cloudwatch.lambda_handler```
    * Create a new basic execution role (your IAM user must have sufficient permissions to create & assign new roles)
 
 6. Set Environment Variables:
-   * Token value should match UUID provided by Logentries UI or API
-   * Region should be that of your LE account - currently only ```eu```
 
-   | Key       | Value      |
-   |-----------|------------|
-   | region    | eu         |
-   | token     | token uuid |
+   | Key       | Value      | Comment  |
+   |-----------|------------|----------|
+   | token     | token uuid | The UUID provided by Logentries UI or API - required |
+   | region    | eu         | That of your LE account - currently only `eu` |
+   | prefix    | true|false | Prefix lines with cloudwatch log group and stream information - default false |
+   | include   | re         | if provided only lines matching this regex are included to LE |
 
 7. Allocate resources:
    * Set memory to 128 MB

certifi/__init__.py

@@ -1,3 +1,3 @@
-from .core import where, old_where
+from .core import where
 
-__version__ = "2016.09.26"
+__version__ = "2018.11.29"

certifi/core.py

@@ -8,29 +8,13 @@
 This module returns the installation location of cacert.pem.
 """
 import os
-import warnings
-
-
-class DeprecatedBundleWarning(DeprecationWarning):
-    """
-    The weak security bundle is being deprecated. Please bother your service
-    provider to get them to stop using cross-signed roots.
-    """
 
 
 def where():
-    f = os.path.split(__file__)[0]
+    f = os.path.dirname(__file__)
 
     return os.path.join(f, 'cacert.pem')
 
 
-def old_where():
-    warnings.warn(
-        "The weak security bundle is being deprecated.",
-        DeprecatedBundleWarning
-    )
-    f = os.path.split(__file__)[0]
-    return os.path.join(f, 'weak.pem')
-
 if __name__ == '__main__':
     print(where())

le_cloudwatch.py

@@ -1,5 +1,6 @@
 import logging
 import json
+import re
 import gzip
 import socket
 import ssl
@@ -13,35 +14,80 @@
 
 logger.info('Loading function...')
 
-REGION = os.environ.get('region')
+REGION = os.environ.get('region') or 'eu'
 ENDPOINT = '{}.data.logs.insight.rapid7.com'.format(REGION)
 PORT = 20000
 TOKEN = os.environ.get('token')
-LINE = u'\u2028'.encode('utf-8')
+PREFIX_LINES = os.environ.get('prefix') in ('true', 'yes')
+INCLUDE = os.environ.get('include')
+LINE_SEPARATOR = u'\u2028'.encode('utf-8')
+
+include_if = None
+if INCLUDE:
+    include_if = re.compile(INCLUDE)
+
 
 def treat_message(message):
-    return message.replace('\n', LINE)
+    return message.strip(' \n').replace('\n', LINE_SEPARATOR)
+
+
+def send_lines(sock, cw_data_dict):
+    # Optionally get a "<functionname stream> " prefix on all lines so can see
+    # what logged each line. E.G. extract from cw_data_dict = dict(
+    #    logGroup="/aws/lambda/hello-world-test",
+    #    logStream="2018/01/04/[$LATEST]bdb3a48bb55c404398b46ef71881d602")
+    prefix = ''
+    if PREFIX_LINES:
+        # only use last part if slash delimited and last 7 significant enough
+        prefix = '<%s %s> ' % (
+            cw_data_dict['logGroup'].split('/')[-1],
+            cw_data_dict['logStream'][-7:]
+        )
+
+    def send(logentry):
+        sock.sendall('%s %s%s\n' % (
+            TOKEN,
+            prefix,
+            treat_message(logentry))
+        )
+
+    # loop through the log events and send to the endpoint
+    count = 0
+    for log_event in cw_data_dict['logEvents']:
+        # Note that log_event['timestamp'] is not used
+        extractedFields = log_event.get('extractedFields', None)
+        if extractedFields:
+            message = json.dumps(extractedFields)
+        else:
+            message = log_event['message']
+
+        if not include_if or include_if.match(message):
+            send(message)
+            count += 1
+
+    total = len(cw_data_dict['logEvents'])
+    if include_if:
+        logger.info('Sent %d/%d log events', count, total)
+    else:
+        logger.info('Sent %s log events', total)
 
 
 def lambda_handler(event, context):
-    sock = create_socket()
+    logger.info('Received log stream...')
 
     if not validate_uuid(TOKEN):
         logger.critical('{} is not a valid token. Exiting.'.format(TOKEN))
         raise SystemExit
-    else:
-        cw_data = str(event['awslogs']['data'])
-        cw_logs = gzip.GzipFile(fileobj=StringIO(cw_data.decode('base64', 'strict'))).read()
-        log_events = json.loads(cw_logs)
-        logger.info('Received log stream...')
-        for log_event in log_events['logEvents']:
-            # look for extracted fields, if not present, send plain message
-            try:
-                sock.sendall('{} {}\n'.format(TOKEN, json.dumps(log_event['extractedFields'])))
-            except KeyError:
-                sock.sendall('{} {}\n'.format(TOKEN, treat_message(log_event['message'])))
 
+    cw_data = str(event['awslogs']['data'])
+    cw_data_file = StringIO(cw_data.decode('base64', 'strict'))
+    cw_data_json = gzip.GzipFile(fileobj=cw_data_file).read()
+    cw_data_dict = json.loads(cw_data_json)
+
+    sock = create_socket()
+    send_lines(sock, cw_data_dict)
     sock.close()
+
     logger.info('Function execution finished.')