diff --git a/.gitignore b/.gitignore index 1c20e402..ef8b881c 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,6 @@ venv.bak/ # mypy .mypy_cache/ + +#restconfig +.restconfig.json \ No newline at end of file diff --git a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py index 6db90c0c..644b4627 100644 --- a/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py +++ b/examples/generate_html_notices_report_from_json/generate_html_notices_from_json.py @@ -1,24 +1,82 @@ from jinja2 import Environment, FileSystemLoader +from slugify import slugify import os import json import argparse +import datetime +import re parser = argparse.ArgumentParser("A program to transform a JSON formatted representation of a Black Duck notices file into a standalone HTML document") parser.add_argument("json_file", help="The input JSON file to be converted") parser.add_argument("output_file_html", help="The file to output the results to") args = parser.parse_args() +date = datetime.date.today() templates_dir = os.path.dirname(os.path.abspath(__file__)) env = Environment(loader=FileSystemLoader(templates_dir)) +env.filters['slugify'] = slugify template = env.get_template('notices-template.html') +copyrightFilters = [ + re.compile(r".*[0-9]{4,}.*"), # at least four numbers for years. years with 2 digits tend to follow 4 (e.g., 1991, 92, 93) + # an email address https://stackoverflow.com/a/201378 + re.compile(r'''.*(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]).*'''), + re.compile(r"copyright", re.IGNORECASE), # an explicit copyright statement +] -with open(args.output_file_html, 'w+') as fh: +def processCopyrightText(copyrightTexts: list, filters:list=[], noticeComponents:list=[]) -> dict: + """Sort, filter, and remove duplicates from copyright texts from componentCopyrightTexts. + + The algorithm for Copyright Texts in Black Duck are very conservative and result in a lot of potentially false positives. This function will attempt to remove false positives and order the list. + + Args: + copyrightTexts (list): Dictionary with list of copyrights in key copyrightTexts + filters (list, optional): List of regex pattern filters or functions. Matching any filters will keep the item. Defaults to []. + noticeComponents (list, optional): List of componentLicenses that are in the notices json. Defaults to []. + + Returns: + dict: A processed version of copyrightTexts. + """ + from collections import OrderedDict + componentDict = {} + result=[] + if noticeComponents: + for component in noticeComponents: + componentDict[component["component"]["projectName"]]=component["component"].get("versionName") + if copyrightTexts: + for component in copyrightTexts: + componentName = component["componentVersionSummary"]["projectName"] + componentVersion = component["componentVersionSummary"].get("versionName") + if not(componentDict.get(componentName) and componentVersion == componentDict.get(componentName)): + continue + copyrightlist = component.get("copyrightTexts") + if filters: + filteredlist = [] + testFunc = None + for filterItem in filters: + if isinstance(filterItem, re.Pattern): + testFunc = lambda x: filterItem.match(x) + elif isinstance(filterItem, function): + testFunc = filterItem + if testFunc: + filteredlist += list(filter(testFunc, copyrightlist)) + copyrightlist = map(lambda x: x.strip(),filteredlist) + if copyrightlist: + component["copyrightTexts"] = list(OrderedDict.fromkeys(copyrightlist)) + component["copyrightTexts"].sort() + result.append(component) + result.sort(key=lambda item: (item["componentProjectName"].lower(), + item["componentVersionSummary"]["versionName"].lower())) + return result + +with open(args.output_file_html, 'wb+') as fh: with open(args.json_file, 'r') as lj: data = json.load(lj) fileContent = data['reportContent'][0]['fileContent'] fh.write(template.render(componentLicenses=fileContent['componentLicenses'], licenseTexts=fileContent['licenseTexts'], - componentCopyrightTexts=fileContent['componentCopyrightTexts'], - projectVersion=fileContent['projectVersion'])) \ No newline at end of file + componentCopyrightTexts=processCopyrightText(fileContent['componentCopyrightTexts'], copyrightFilters, fileContent['componentLicenses']), + projectVersion=fileContent['projectVersion'], + date=date + ).encode("utf-8")) diff --git a/examples/generate_html_notices_report_from_json/head.html b/examples/generate_html_notices_report_from_json/head.html new file mode 100644 index 00000000..be553f8c --- /dev/null +++ b/examples/generate_html_notices_report_from_json/head.html @@ -0,0 +1,3 @@ + + {{ projectVersion.projectName }} — {{ projectVersion.versionName }} — Notices File + \ No newline at end of file diff --git a/examples/generate_html_notices_report_from_json/intro.html b/examples/generate_html_notices_report_from_json/intro.html new file mode 100644 index 00000000..52a56177 --- /dev/null +++ b/examples/generate_html_notices_report_from_json/intro.html @@ -0,0 +1,14 @@ +

+ {{ projectVersion.projectName }} — {{ projectVersion.versionName }} — Notices File +

+
+
+ Phase: + {{ projectVersion.versionPhase }} +
+ | +
+ Distribution: + {{ projectVersion.versionDistribution }} +
+
\ No newline at end of file diff --git a/examples/generate_html_notices_report_from_json/notices-template.html b/examples/generate_html_notices_report_from_json/notices-template.html index 64496679..d9aa5859 100644 --- a/examples/generate_html_notices_report_from_json/notices-template.html +++ b/examples/generate_html_notices_report_from_json/notices-template.html @@ -1,27 +1,15 @@ +{% include 'head.html' %} -

- {{ projectVersion.projectName }} — {{ projectVersion.versionName }} — Notices File -

-
-
- Phase: - {{ projectVersion.versionPhase }} -
- | -
- Distribution: - {{ projectVersion.versionDistribution }} -
-
+{% include 'intro.html' %} -

Components

+

Components ({{ componentLicenses | length}})

- - + + @@ -29,18 +17,29 @@

Components

@@ -50,12 +49,16 @@

Components

ComponentLicenseComponent (click for copyrights)License (click for license)
- {{ row.component.projectName }} + {% set componentCopyrightFound = (componentCopyrightTexts|length > 0 and + componentCopyrightTexts | selectattr("componentVersionSummary.projectName", "equalto", row.component.projectName) | selectattr("componentVersionSummary.versionName", "equalto", row.component.versionName) | list | length > 0 ) %} + {% if componentCopyrightFound %} + + {% endif %} + + {{ row.component.projectName }} {% if row.component.versionName %} {{ row.component.versionName }} {% endif %} + {% if componentCopyrightFound %} + + {% endif %}
{% if row.licenses|length > 0 %} + {{ row.licenses[0]['name'] }} + {% endif %}
{% if componentCopyrightTexts|length > 0 %} -

Copyright Data

+

Copyright Data ({{ componentCopyrightTexts|length }})

{% endif %} -

Licenses

-
+

Licenses ({{licenseTexts | length}})

+
{% for row in licenseTexts %} -

{{ row.name }}

-
- {% for comp in row.components %} - {{ comp.projectName }} {{ comp.versionName }}{% if not loop.last %},{% endif %} - {% endfor %} -
- 
{{ row.text|e }}
+ {% set rowLength = row.components | length %} + {% if rowLength <= 1 %} + {% set rowName = "%s - %s %s" | format(row.name,row.components[0].projectName,row.components[0].versionName) %} + {% else %} + {% set rowName = "%s - %s components"| format(row.name,rowLength) %} + {% endif %} + {% if rowLength > 1 %} +

{{ rowName }}

+
+
+
    + {% for comp in row.components %} +
  • + {{ comp.projectName }} {{ comp.versionName }}
    • + {% endfor %} +
+
+
+ {% else %} +

{{ rowName }}

+ {% endif %} + 
{{ row.text|e }}
{% endfor %}
diff --git a/examples/generate_html_notices_report_from_json/requirements.txt b/examples/generate_html_notices_report_from_json/requirements.txt index 24db46ed..c0a52969 100644 --- a/examples/generate_html_notices_report_from_json/requirements.txt +++ b/examples/generate_html_notices_report_from_json/requirements.txt @@ -1,2 +1,3 @@ # for use of the template to generate the html jinja2 +python-slugify \ No newline at end of file