rel 2023.4

Author: FredHappyface

CHANGELOG.md

@@ -3,6 +3,12 @@
 All major and minor version changes will be documented in this file. Details of
 patch-level version changes can be found in [commit messages](../../commits/master).
 
+## 2023.4 - 2023/08/20
+
+- Refactor and code enhancements
+- Fixed Bug: fix behaviour of dependency discovery for 'extras', thank you https://github.com/TuriJ95!
+- Fixed Bug: ignore-package not working, thank you https://github.com/mathiasbockwoldt !
+
 ## 2023.3 - 2023/07/29
 
 - Fixed Bug: requirements:requirements.txt reading mode, thank you https://github.com/NicolaDonelli

documentation/reference/licensecheck/formatter.md

@@ -178,6 +178,4 @@ Strip ansi codes from a given string
 ```python
 def stripAnsi(string: str) -> str:
     ...
-```
-
-
+```

documentation/reference/licensecheck/get_deps.md

@@ -12,7 +12,7 @@ Get Deps
 
 ## getDepsWithLicenses
 
-[Show source in get_deps.py:129](../../../licensecheck/get_deps.py#L129)
+[Show source in get_deps.py:157](../../../licensecheck/get_deps.py#L157)
 
 Get a set of dependencies with licenses and determine license compatibility.
 
@@ -53,7 +53,7 @@ def getDepsWithLicenses(
 
 ## getReqs
 
-[Show source in get_deps.py:19](../../../licensecheck/get_deps.py#L19)
+[Show source in get_deps.py:20](../../../licensecheck/get_deps.py#L20)
 
 Get requirements for the end user project/ lib.
 
@@ -83,6 +83,4 @@ def getReqs(using: str) -> set[ucstr]:
 
 #### See also
 
-- [ucstr](./types.md#ucstr)
-
-
+- [ucstr](./types.md#ucstr)

documentation/reference/licensecheck/license_matrix.md

@@ -132,6 +132,4 @@ def licenseType(lice: ucstr, ignoreLicenses: list[ucstr] | None = None) -> list[
 #### See also
 
 - [License](./types.md#license)
-- [ucstr](./types.md#ucstr)
-
-
+- [ucstr](./types.md#ucstr)

documentation/reference/licensecheck/module.md

@@ -5,5 +5,4 @@
 Module
 
 > Auto-generated documentation for [licensecheck.__main__](../../../licensecheck/__main__.py) module.
-
 - [Module](#module)

documentation/reference/licensecheck/packageinfo.md

@@ -204,6 +204,4 @@ def licenseFromClassifierlist(classifiers: list[str]) -> ucstr:
 
 #### See also
 
-- [ucstr](./types.md#ucstr)
-
-
+- [ucstr](./types.md#ucstr)

documentation/reference/licensecheck/types.md

@@ -14,7 +14,7 @@ Types
 
 ## License
 
-[Show source in types.py:45](../../../licensecheck/types.py#L45)
+[Show source in types.py:46](../../../licensecheck/types.py#L46)
 
 #### Attributes
 
@@ -50,7 +50,7 @@ class License(Enum):
 
 ## PackageInfo
 
-[Show source in types.py:28](../../../licensecheck/types.py#L28)
+[Show source in types.py:29](../../../licensecheck/types.py#L29)
 
 PackageInfo type.
 
@@ -65,7 +65,7 @@ class PackageInfo:
 
 ## ucstr
 
-[Show source in types.py:16](../../../licensecheck/types.py#L16)
+[Show source in types.py:17](../../../licensecheck/types.py#L17)
 
 Uppercase string
 
@@ -80,7 +80,7 @@ class ucstr(str):
 
 ## printLicense
 
-[Show source in types.py:88](../../../licensecheck/types.py#L88)
+[Show source in types.py:89](../../../licensecheck/types.py#L89)
 
 Output a license as plain text
 
@@ -102,6 +102,4 @@ def printLicense(licenseEnum: L) -> str:
 
 #### See also
 
-- [L](#l)
-
-
+- [L](#l)

licensecheck/get_deps.py

@@ -7,9 +7,9 @@
 from pathlib import Path
 from typing import Any
 
-import pkg_resources
-import requirements
 import tomli
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 
 from licensecheck import license_matrix, packageinfo
 from licensecheck.types import JOINS, License, PackageInfo, session, ucstr
@@ -56,11 +56,25 @@ def getReqs(using: str) -> set[ucstr]:
 def _doGetReqs(
 	using: str, extras: str | None, pyproject: dict[str, Any], requirementsPaths: list[Path]
 ) -> set[ucstr]:
-	resolveReq = lambda req: ucstr(pkg_resources.Requirement.parse(req).project_name)
-	resolveExtraReq = lambda extra_req: re.sub("extra == ", "", re.findall(r"extra == '.*?'",extra_req)[0].replace("'", "")) if len(re.findall(r"extra == '.*?'",extra_req)) > 0 else None
 
 	reqs = set()
-	extras_reqs = dict()
+	extrasReqs = {}
+
+	def resolveReq(req: str, extra: bool = True) -> ucstr:
+		requirement = Requirement(req)
+		extras = {ucstr(extra) for extra in requirement.extras}
+		name = ucstr(canonicalize_name(requirement.name))
+		canonicalName = name
+		if len(extras) > 0:
+			canonicalName = ucstr(f"{name}[{list(extras)[0]}]")
+			extrasReqs[name] = extras
+		return canonicalName if extra else name
+
+	def resolveExtraReq(extraReq: str) -> ucstr | None:
+		match = re.search(r"extra\s*==\s*'(.*?)'", extraReq)
+		if match is None:
+			return None
+		return ucstr(match.group(1))
 
 	if using == "poetry":
 		try:
@@ -78,12 +92,7 @@ def _doGetReqs(
 			reqLists.append(project.get("dev-dependencies", {}))
 		for reqList in reqLists:
 			for req in reqList:
-				try:
-					extra = re.search(r'(?<=\[)(.*?)(?=\])',req).group(0)
-					extras_reqs[resolveReq(req)] = extra
-					reqs.add(f"{resolveReq(req)}[{extra}]")
-				except (KeyError, AttributeError):
-					reqs.add(resolveReq(req))
+				reqs.add(resolveReq(req))
 	# PEP631 (hatch)
 	if using == "PEP631":
 		try:
@@ -97,27 +106,19 @@ def _doGetReqs(
 			reqLists.extend(project["optional-dependencies"][x] for x in extras.split(";"))
 		for reqList in reqLists:
 			for req in reqList:
-				try:
-					extra = re.search(r'(?<=\[)(.*?)(?=\])',req).group(0)
-					extras_reqs[resolveReq(req)] = extra
-					reqs.add(f"{resolveReq(req)}[{extra}]")
-				except (KeyError, AttributeError):
-					reqs.add(resolveReq(req))
+				reqs.add(resolveReq(req))
 
 	# Requirements
 	if using == "requirements":
 		for reqPath in requirementsPaths:
 			if not reqPath.exists():
 				raise RuntimeError(f"Could not find specification of requirements ({reqPath}).")
 
-			with open(reqPath, encoding="utf-8") as requirementsTxt:
-				for req in requirements.parse(requirementsTxt):
-					if len(req.extras)>0:
-						extras_reqs[resolveReq(req.name)] = req.extras
-						for extra in req.extras:
-							reqs.add(f"{resolveReq(req.name)}[{extra}]")
-					else:
-						reqs.add(resolveReq(req.name))
+			for line in reqPath.read_text(encoding="utf-8").splitlines():
+				line = line.strip()
+				if not line or line[0] in {"#", "-"}:
+					continue
+				reqs.add(resolveReq(line))
 
 	try:
 		reqs.remove("PYTHON")
@@ -132,19 +133,26 @@ def _doGetReqs(
 			for req in [resolveReq(req) for req in pkgMetadata.get_all("Requires-Dist") or []]:
 				requirementsWithDeps.add(req)
 		except metadata.PackageNotFoundError:
-			request = session.get(f"https://pypi.org/pypi/{requirement.split('[')[0]}/json", timeout=60)
+			request = session.get(
+				f"https://pypi.org/pypi/{requirement.split('[')[0]}/json", timeout=60
+			)
 			response = request.json()
 			try:
 				for dependency in response["info"]["requires_dist"]:
-					if resolveExtraReq(dependency) is not None:
-						if (resolveReq(requirement) in extras_reqs.keys()) and (resolveExtraReq(dependency) in extras_reqs[resolveReq(requirement)]):
-							requirementsWithDeps.add(resolveReq(dependency))
+					dep = resolveReq(dependency, False)
+					req = resolveReq(requirement, False)
+					extra = resolveExtraReq(dependency)
+					if extra is not None:
+						if req in extrasReqs and extra in extrasReqs.get(req, []):
+							requirementsWithDeps.add(dep)
+						# else: pass
 					else:
-						requirementsWithDeps.add(resolveReq(dependency))
+						requirementsWithDeps.add(dep)
 			except (KeyError, TypeError):
 				pass
 
-	return {r.split('[')[0] for r in requirementsWithDeps}
+	return {r.split("[")[0] for r in requirementsWithDeps}
+
 
 def getDepsWithLicenses(
 	using: str,

licensecheck/types.py

@@ -6,9 +6,10 @@
 from enum import Enum
 from pathlib import Path
 
+import requests_cache
+
 THISDIR = Path(__file__).resolve().parent
 
-import requests_cache
 
 session = requests_cache.CachedSession(f"{THISDIR}/licensecheck")
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "licensecheck"
-version = "2023.3"
+version = "2023.4"
 license = "mit"
 description = "Output the licenses used by dependencies and check if these are compatible with the project license"
 authors = ["FredHappyface"]
@@ -28,12 +28,12 @@ licensecheck = 'licensecheck:cli'
 
 [tool.poetry.dependencies]
 python = "^3.8"
-requirements-parser = "<2,>=0.5.0"
 requests = "<3,>=2.31.0"
 fhconfparser = "<2024,>=2022"
 tomli = "<3,>=2.0.1"
 rich = "<14,>=13.4.2"
 requests-cache = "<2,>=1.1.0"
+packaging = "<24,>=23.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.1"

requirements.txt

@@ -1,6 +1,6 @@
 fhconfparser<2024,>=2022
+packaging<24,>=23.1
 requests-cache<2,>=1.1.0
 requests<3,>=2.31.0
-requirements-parser<2,>=0.5.0
 rich<14,>=13.4.2
 tomli<3,>=2.0.1

tests/test_get_deps.py

@@ -15,47 +15,61 @@ def test_doGetReqs_PEP631():
 	requirementsPaths = []
 
 	assert get_deps._doGetReqs(using, extras, pyproject, requirementsPaths) == {
-		'DOCKERPTY',
-		'PACKAGING',
-		'ATTRS',
-		'JSONSCHEMA',
-		'PYYAML',
-		'PYSOCKS',
-		'CERTIFI',
-		'ENUM34',
-		'DOCKER',
-		'TEXTTABLE',
-		'PYWIN32',
-		'JSONSCHEMA-SPECIFICATIONS',
-		'IPADDRESS',
-		'PKGUTIL-RESOLVE-NAME',
-		'DOCOPT',
-		'BACKPORTS.SSL-MATCH-HOSTNAME',
-		'PARAMIKO',
-		'IDNA',
-		'COLORAMA',
-		'IMPORTLIB-RESOURCES',
-		'CACHED-PROPERTY',
-		'DISTRO',
-		'BACKPORTS.SHUTIL-GET-TERMINAL-SIZE',
-		'CHARSET-NORMALIZER',
-		'URLLIB3',
-		'WEBSOCKET-CLIENT',
-		'RPDS-PY',
-		'SUBPROCESS32',
-		'REQUESTS',
-		'REFERENCING',
-		'CHARDET',
-		'PYTHON-DOTENV'
+		"DOCKERPTY",
+		"PACKAGING",
+		"ATTRS",
+		"JSONSCHEMA",
+		"PYYAML",
+		"PYSOCKS",
+		"CERTIFI",
+		"ENUM34",
+		"DOCKER",
+		"TEXTTABLE",
+		"PYWIN32",
+		"JSONSCHEMA-SPECIFICATIONS",
+		"IPADDRESS",
+		"PKGUTIL-RESOLVE-NAME",
+		"DOCOPT",
+		"BACKPORTS-SSL-MATCH-HOSTNAME",
+		"PARAMIKO",
+		"IDNA",
+		"COLORAMA",
+		"IMPORTLIB-RESOURCES",
+		"CACHED-PROPERTY",
+		"DISTRO",
+		"BACKPORTS-SHUTIL-GET-TERMINAL-SIZE",
+		"CHARSET-NORMALIZER",
+		"URLLIB3",
+		"WEBSOCKET-CLIENT",
+		"RPDS-PY",
+		"SUBPROCESS32",
+		"REQUESTS",
+		"REFERENCING",
+		"CHARDET",
+		"PYTHON-DOTENV",
 	}
 
+
 def test_doGetReqs_requirements():
 
 	using = "requirements"
 	extras = f"{THISDIR}/data/test_requirements.txt"
 	pyproject = {}
 	requirementsPaths = [Path(f"{THISDIR}/data/test_requirements.txt")]
 	deps = get_deps._doGetReqs(using, extras, pyproject, requirementsPaths)
-	assert deps == {'NUMPY', 'ODFPY', 'OPENPYXL', 'PANDAS', 'PYTHON-DATEUTIL', 'PYTZ', 'PYXLSB', 'TZDATA', 'XLRD', 'XLSXWRITER'}
-	assert 'OPENPYXL' in deps
-	assert 'XARRAY' not in deps #xarray is an optional dependecy of pandas associated with 'computation' key that is not tracked in test_requirements.txt
+	assert deps == {
+		"NUMPY",
+		"ODFPY",
+		"OPENPYXL",
+		"PANDAS",
+		"PYTHON-DATEUTIL",
+		"PYTZ",
+		"PYXLSB",
+		"TZDATA",
+		"XLRD",
+		"XLSXWRITER",
+	}
+	assert "OPENPYXL" in deps
+	assert (
+		"XARRAY" not in deps
+	)  # xarray is an optional dependecy of pandas associated with 'computation' key that is not tracked in test_requirements.txt