Improve generate_html_notices_from_json.py

.gitignore

@@ -103,3 +103,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+#restconfig
+.restconfig.json

examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py

@@ -1,24 +1,82 @@
 from jinja2 import Environment, FileSystemLoader
+from slugify import slugify
 import os
 import json
 import argparse
+import datetime
+import re
 
 parser = argparse.ArgumentParser("A program to transform a JSON formatted representation of a Black Duck notices file into a standalone HTML document")
 parser.add_argument("json_file", help="The input JSON file to be converted")
 parser.add_argument("output_file_html", help="The file to output the results to")
 
 args = parser.parse_args()
+date = datetime.date.today()
 
 templates_dir = os.path.dirname(os.path.abspath(__file__))
 env = Environment(loader=FileSystemLoader(templates_dir))
+env.filters['slugify'] = slugify
 template = env.get_template('notices-template.html')
+copyrightFilters = [
+    re.compile(r".*[0-9]{4,}.*"), # at least four numbers for years. years with 2 digits tend to follow 4 (e.g., 1991, 92, 93)
+    # an email address https://stackoverflow.com/a/201378
+    re.compile(r'''.*(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]).*'''), 
+    re.compile(r"copyright", re.IGNORECASE), # an explicit copyright statement
+]
 
-with open(args.output_file_html, 'w+') as fh:
+def processCopyrightText(copyrightTexts: list, filters:list=[], noticeComponents:list=[]) -> dict:
+    """Sort, filter, and remove duplicates from copyright texts from componentCopyrightTexts.
+
+    The algorithm for Copyright Texts in Black Duck are very conservative and result in a lot of potentially false positives. This function will attempt to remove false positives and order the list.
+
+    Args:
+        copyrightTexts (list): Dictionary with list of copyrights in key copyrightTexts
+        filters (list, optional): List of regex pattern filters or functions. Matching any filters will keep the item. Defaults to [].
+        noticeComponents (list, optional): List of componentLicenses that are in the notices json. Defaults to [].
+
+    Returns:
+        dict: A processed version of copyrightTexts.
+    """
+    from collections import OrderedDict
+    componentDict = {}
+    result=[]
+    if noticeComponents:
+        for component in noticeComponents:
+            componentDict[component["component"]["projectName"]]=component["component"].get("versionName")
+    if copyrightTexts:
+        for component in copyrightTexts:
+            componentName = component["componentVersionSummary"]["projectName"]
+            componentVersion = component["componentVersionSummary"].get("versionName")
+            if not(componentDict.get(componentName) and componentVersion == componentDict.get(componentName)):
+                continue
+            copyrightlist = component.get("copyrightTexts")
+            if filters:
+                filteredlist = []
+                testFunc = None
+                for filterItem in filters:
+                    if isinstance(filterItem, re.Pattern):
+                        testFunc = lambda x: filterItem.match(x)
+                    elif isinstance(filterItem, function):
+                        testFunc = filterItem
+                    if testFunc:
+                        filteredlist += list(filter(testFunc, copyrightlist))                  
+                copyrightlist = map(lambda x: x.strip(),filteredlist)
+            if copyrightlist:
+                component["copyrightTexts"] = list(OrderedDict.fromkeys(copyrightlist))
+                component["copyrightTexts"].sort()
+            result.append(component)
+    result.sort(key=lambda item: (item["componentProjectName"].lower(), 
+                                item["componentVersionSummary"]["versionName"].lower()))
+    return result
+
+with open(args.output_file_html, 'wb+') as fh:
     with open(args.json_file, 'r') as lj:
         data = json.load(lj)
         fileContent = data['reportContent'][0]['fileContent']
 
         fh.write(template.render(componentLicenses=fileContent['componentLicenses'],
                                  licenseTexts=fileContent['licenseTexts'],
-                                 componentCopyrightTexts=fileContent['componentCopyrightTexts'],
-                                 projectVersion=fileContent['projectVersion']))
+                                 componentCopyrightTexts=processCopyrightText(fileContent['componentCopyrightTexts'], copyrightFilters, fileContent['componentLicenses']),
+                                 projectVersion=fileContent['projectVersion'],
+                                 date=date
+                                 ).encode("utf-8"))

examples/generate_html_notices_report_from_json/head.html

@@ -0,0 +1,3 @@
+<head>
+    <title>{{ projectVersion.projectName }} &mdash; {{ projectVersion.versionName }} &mdash; Notices File
+    </title></head>

examples/generate_html_notices_report_from_json/intro.html

@@ -0,0 +1,14 @@
+<h1>
+    {{ projectVersion.projectName }} <span>&mdash;</span> {{ projectVersion.versionName }} <span>&mdash;</span> Notices File
+</h1>
+<div style="clear:both;display:block;">
+    <div style="display:inline-block;">
+        <span>Phase:</span>
+        <span>{{ projectVersion.versionPhase }}</span>
+    </div>
+    |
+    <div style="display:inline-block;">
+        <span>Distribution:</span>
+        <span>{{ projectVersion.versionDistribution }}</span>
+    </div>
+</div>

examples/generate_html_notices_report_from_json/notices-template.html

@@ -1,46 +1,45 @@
 <!DOCTYPE html>
 <html style="font-family:Open Sans,sans-serif;">
+{% include 'head.html' %}
 <body>
-<h1>
-    {{ projectVersion.projectName }} <span>&mdash;</span> {{ projectVersion.versionName }} <span>&mdash;</span> Notices File
-</h1>
-<div style="clear:both;display:block;">
-    <div style="display:inline-block;">
-        <span>Phase:</span>
-        <span>{{ projectVersion.versionPhase }}</span>
-    </div>
-    |
-    <div style="display:inline-block;">
-        <span>Distribution:</span>
-        <span>{{ projectVersion.versionDistribution }}</span>
-    </div>
-</div>
+{% include 'intro.html' %}
 
-<h1 id="comp-header" style="cursor:pointer;">Components</h1>
+<h1 id="comp-header" style="cursor:pointer;">Components ({{ componentLicenses | length}})</h1>
 <table style="width:100%;border-collapse:collapse;font-family:Open Sans,sans-serif;" id="comp-container">
     <thead>
     <tr style="border-bottom:1px solid black">
-        <th style="text-align:left;background-color:#f6f6f6;font-weight:500">Component</th>
-        <th style="text-align:left;background-color:#f6f6f6;font-weight:500">License</th>
+        <th style="text-align:left;background-color:#f6f6f6;font-weight:500">Component (click for copyrights)</th>
+        <th style="text-align:left;background-color:#f6f6f6;font-weight:500">License (click for license)</th>
     </tr>
     </thead>
     <tbody>
     {% for row in componentLicenses %}
         <tr style="border-bottom:1px solid black">
             <td>
                 <span>
-                    <span style="font-weight:700">{{ row.component.projectName }}</span>
+                {%  set componentCopyrightFound = (componentCopyrightTexts|length > 0 and 
+                    componentCopyrightTexts | selectattr("componentVersionSummary.projectName", "equalto", row.component.projectName) | selectattr("componentVersionSummary.versionName", "equalto", row.component.versionName) | list | length > 0 ) %}
+                {%  if componentCopyrightFound %}
+                    <a href="#{{ "%s-%s-%s"|format("copyright",row.component.projectName,row.component.versionName) | slugify }}">
+                {% endif %}
+                        <span style="font-weight:700">
+                            {{ row.component.projectName }}</span>
                     <span style="font-weight:500">
                         {% if row.component.versionName %}
                             {{ row.component.versionName }}
                         {% endif %}
                     </span>
+                {%  if componentCopyrightFound %}
+                    </a>
+                {% endif %}
                 </span>
             </td>
             <td>
                 <div>
                     {% if row.licenses|length > 0 %}
+                    <a href="#{{ "%s-%s"|format(row.component.projectName,row.component.versionName) | slugify }}">
                         {{ row.licenses[0]['name'] }}
+                    </a>
                     {% endif %}
                 </div>
             </td>
@@ -50,74 +49,76 @@ <h1 id="comp-header" style="cursor:pointer;">Components</h1>
 </table>
 
 {% if componentCopyrightTexts|length > 0 %}
-    <h1 id="copyright-header" style="cursor:pointer;">Copyright Data</h1>
+    <h1 id="copyright-header" style="cursor:pointer;">Copyright Data ({{ componentCopyrightTexts|length }})</h1>
     <div id="copyright-container">
         {% for row in componentCopyrightTexts %}
-            <h2>
-                {{ row.componentVersionSummary.projectName }} <span>&mdash;</span> {{ row.originFullName }}
-            </h2>
+        <h2 id="{{ "%s-%s-%s"|format("copyright",row.componentVersionSummary.projectName,row.componentVersionSummary.versionName) | slugify}}">
+            {{ row.componentVersionSummary.projectName }} <span>&mdash;</span> {{ row.originFullName }}
+        </h2>
+        <details open>
+            <summary>
+                Details - {{ row.copyrightTexts | length}}
+            </summary>
             {% for cr in row.copyrightTexts %}
                 <ul>
                     <li style="list-style-type:disc;">
                         {{ cr|e }}
                     </li>
                 </ul>
             {% endfor %}
+        </details>
         {% endfor %}
     </div>
 {% endif %}
 
-<h1 id="license-header" style="cursor:pointer;">Licenses</h1>
-<div id="license-container">
+<h1 id="license-header" style="cursor:pointer;">Licenses ({{licenseTexts | length}}) </h1>
+<div id="license-container">  
     {% for row in licenseTexts %}
-        <h2>{{ row.name }}</h2>
-        <div>
-            {% for comp in row.components %}
-                {{ comp.projectName }} {{ comp.versionName }}{% if not loop.last %},{% endif %}
-            {% endfor %}
-        </div>
-        <pre style="background-color:#f6f6f6;border:1px solid black">{{ row.text|e }}</pre>
+        {%  set rowLength = row.components | length %}
+        {% if rowLength <= 1 %}
+            {%  set rowName = "%s - %s %s" | format(row.name,row.components[0].projectName,row.components[0].versionName) %}
+        {% else %}
+            {%  set rowName = "%s - %s components"| format(row.name,rowLength) %}
+        {% endif %}
+        {% if rowLength > 1 %}
+            <h2>{{ rowName }}</h2>
+            <details open>
+            <div>
+                <ul>
+                {% for comp in row.components %}
+                    <li id="{{ "%s-%s"|format(comp.projectName,comp.versionName) | slugify }}" style="list-style-type:disc;">
+                        {{ comp.projectName }} {{ comp.versionName }}</li>
+                {% endfor %}
+                </ul>
+            </div>
+            </details>
+        {% else %}
+            <h2 id="{{ "%s-%s"|format(row.components[0].projectName,row.components[0].versionName) | slugify }}">{{ rowName }}</h2>
+        {% endif %}
+    <pre style="background-color:#f6f6f6;border:1px solid black">{{ row.text|e }}</pre>
     {% endfor %}
 </div>
 
 
 
 <script>
-    compheader = document.getElementById("comp-header")
-    if (!(compheader === null)) {
-        compheader.addEventListener("click", function() {
-            container = document.getElementById("comp-container")
+    function makeCollapseable(item){
+        header = document.getElementById(item + "-header")
+        if (!(header === null)) {
+            header.addEventListener("click", function() {
+            container = document.getElementById(item + "-container")
             if (container.style.display === "none") {
                 container.style.display = null;
             } else {
                 container.style.display = "none";
             }
         });
     }
-
-    copyheader = document.getElementById("copyright-header")
-    if (!(copyheader === null)) {
-        copyheader.addEventListener("click", function() {
-            container = document.getElementById("copyright-container")
-            if (container.style.display === "none") {
-                container.style.display = null;
-            } else {
-                container.style.display = "none";
-            }
-        });
     }
 
-    licenseheader = document.getElementById("license-header")
-    if (!(licenseheader === null)) {
-        licenseheader.addEventListener("click", function() {
-            container = document.getElementById("license-container")
-            if (container.style.display === "none") {
-                container.style.display = null;
-            } else {
-                container.style.display = "none";
-            }
-        });
-    }
+    makeCollapseable("comp");
+    makeCollapseable("copyright");
+    makeCollapseable("license");
 
 </script>
 

examples/generate_html_notices_report_from_json/requirements.txt

@@ -1,2 +1,3 @@
 # for use of the template to generate the html
 jinja2
+python-slugify