Publish libraries on smithed (Gamemode4Dev#889)

* Publish libraries to a folder in release

Standalone library build, with needed lantern load files, saved to the release directory to be uploaded to smithed

* Version() support for rich comparison operators

* Modify release script for libraries

* Auto patch tracking for libraries

* Update manifest.py

* Release libraries with module release plugin

* Start to version permalinking

* Complete past-version permalinking

* Fix lantern-load inclusion

* beet-release.yaml plugins for LL

* Library Licenses and Readmes

* Version number appears in lib.zip pack.mcmeta

* Rewrite lib_brewing README

- actually offers explanations for things
- restructures "Additional Uses" section

* Simple typos

* potion tracking clarification

- adds example functions to show how to tag potions and how to execute from landed potions

* custom crafters example

- add loot table for example recipe
- switch example to 1 wool -> 3 string to explain that rounding down is necessary

* Port over example-use packs

* Add smithed pack ids

* Simple library pack icons

Simple inkscape-bitmap-traced icons. Do not conform with the GM4 color map and should be adjusted in the future. I blindly used the colorize filter to achieve this effect

* Copy READMEs to the generated directory

---------

Co-authored-by: BluePsychoRanger <[email protected]>

Author: SpecialBuilder32

beet-release.yaml

@@ -16,12 +16,34 @@ pipeline:
         meta: {copy_files: {data_pack: {LICENSE.md: "../LICENSE.md"}}}
       - gm4.plugins.readme_generator
       - gm4.plugins.write_mcmeta
+      - gm4.plugins.output.release
+    meta:
+      mecha:
+        formatting:
+          layout: preserve
+          nbt_compact: True
+  - broadcast: 'lib_*'
+    extend: 'beet.yaml'
+    pipeline:
+      - beet.contrib.lantern_load.base_data_pack
+      - gm4.plugins.versioning.isolated_library
+      - gm4.plugins.manifest.write_credits
+      - require: [beet.contrib.copy_files]
+        meta:
+          copy_files:
+            data_pack:
+              LICENSE.md: "LICENSE.md"
+              README.md: "README.md"
+              pack.png: "pack.png"
+      - gm4.plugins.module.default_pack_icon
+      - gm4.plugins.readme_generator.libraries
+      - gm4.plugins.write_mcmeta
+      - gm4.plugins.output.release
     meta:
       mecha:
         formatting:
           layout: preserve
           nbt_compact: True
-
 meta:
   autosave:
     link: false

gm4/plugins/manifest.py

@@ -65,7 +65,8 @@ class ManifestCacheModel(BaseModel):
 class ManifestFileModel(BaseModel):
 	"""describes the structure of the meta.json saved to disk"""
 	last_commit: str
-	modules: list[ManifestModuleModel]
+	modules: list[ManifestModuleModel] # straight list for website backward compat
+	libraries: dict[str, ManifestModuleModel]
 	contributors: Any
 
 
@@ -136,59 +137,63 @@ def update_patch(ctx: Context):
 	logger = parent_logger.getChild("update_patch")
 	skin_cache = JsonFile(source_path="gm4/skin_cache.json").data
 
-	modules = ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json).modules
+	manifest_cache = ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
 
 	if manifest_file.exists():
 		manifest = ManifestFileModel.parse_obj(json.loads(manifest_file.read_text()))
 		last_commit = manifest.last_commit
 		released_modules: dict[str, ManifestModuleModel] = {m.id:m for m in manifest.modules if m.version}
+		released_modules |= manifest.libraries
 	else:
 		logger.debug("No existing meta.json manifest file was located")
 		last_commit = None
 		released_modules = {}
 
-	for id in modules:
-		module = modules[id]
-		released = released_modules.get(id, None)
-
-		publish_date = released.publish_date if released else None
-		module.publish_date = publish_date or datetime.datetime.now().date().isoformat()
-
-		deps = _traverse_includes(id) | {"base"}
-		deps_dirs = [element for sublist in [[f"{d}/data", f"{d}/*py"] for d in deps] for element in sublist]
-
-		 # add watches to skins this module uses from other modules. NOTE this could be done in a more extendable way in the future, rather than "hardcoded"
-		skin_dep_dirs: list[str] = []
-		for skin_ref in skin_cache["nonnative_references"].get(id, []):
-			d = skin_cache["skins"][skin_ref]["parent_module"]
-			ns, path = skin_ref.split(":")	
-			skin_dep_dirs.append(f"{d}/data/{ns}/skins/{path}.png")
-		
-		watch_dirs = deps_dirs+skin_dep_dirs
-
-		diff = run(["git", "diff", last_commit, "--shortstat", "--", f"{id}/data", f"{id}/*.py"] + watch_dirs) if last_commit else True
-
-		if not diff and released:
-			# No changes were made, keep the same patch version
-			module.version = released.version
-		elif not released:
-			# First release
-			module.version = module.version.replace("X", "0")
-			logger.debug(f"First release of {id}")
-		else:
-			# Changes were made, bump the patch
-			version = Version(module.version)
-			last_ver = Version(released.version)
+	for packs in (manifest_cache.modules, manifest_cache.libraries):
+		for id in packs:
+			pack = packs[id]
+			released = released_modules.get(id, None)
+
+			publish_date = released.publish_date if released else None
+			pack.publish_date = publish_date or datetime.datetime.now().date().isoformat()
+
+			deps = _traverse_includes(id)
+			if packs is manifest_cache.modules:
+				deps |= {"base"} # scan the base directory if this is a module
+			deps_dirs = [element for sublist in [[f"{d}/data", f"{d}/*py"] for d in deps] for element in sublist]
+
+			# add watches to skins this module uses from other modules. NOTE this could be done in a more extendable way in the future, rather than "hardcoded"
+			skin_dep_dirs: list[str] = []
+			for skin_ref in skin_cache["nonnative_references"].get(id, []):
+				d = skin_cache["skins"][skin_ref]["parent_module"]
+				ns, path = skin_ref.split(":")	
+				skin_dep_dirs.append(f"{d}/data/{ns}/skins/{path}.png")
 
-			if version.minor > last_ver.minor or version.major > last_ver.major: # type: ignore
-				version.patch = 0
+			watch_dirs = deps_dirs+skin_dep_dirs
+
+			diff = run(["git", "diff", last_commit, "--shortstat", "--", f"{id}/data", f"{id}/*.py"] + watch_dirs) if last_commit else True
+
+			if not diff and released:
+				# No changes were made, keep the same patch version
+				pack.version = released.version
+			elif not released:
+				# First release
+				pack.version = pack.version.replace("X", "0")
+				logger.debug(f"First release of {id}")
 			else:
-				version.patch = last_ver.patch + 1 # type: ignore
-				logger.info(f"Updating {id} patch to {version.patch}")
+				# Changes were made, bump the patch
+				version = Version(pack.version)
+				last_ver = Version(released.version)
+				
+				if version.minor > last_ver.minor or version.major > last_ver.major: # type: ignore
+					version.patch = 0
+				else:
+					version.patch = last_ver.patch + 1 # type: ignore
+					logger.info(f"Updating {id} patch to {version.patch}")
 
-			module.version = str(version)
+				pack.version = str(version)
 
-	ctx.cache["gm4_manifest"].json["modules"] = {id:m.dict() for id, m in modules.items()}
+	ctx.cache["gm4_manifest"].json = manifest_cache.dict()
 
 @cache
 def _traverse_includes(project_id: str) -> set[str]:
@@ -213,7 +218,6 @@ def write_meta(ctx: Context):
 	manifest_file = release_dir / "meta.json"
 	manifest = ctx.cache["gm4_manifest"].json.copy()
 	manifest["modules"] = list(manifest["modules"].values()) # convert modules dict down to list for backwards compatability
-	manifest.pop("libraries")
 	manifest.pop("base")
 	manifest_file.write_text(json.dumps(manifest, indent=2))
 
@@ -260,7 +264,7 @@ def write_updates(ctx: Context):
 	if init is None:
 		return
 
-	manifest =ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
+	manifest = ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
 	modules = manifest.modules
 
 	score = f"{ctx.project_id.removeprefix('gm4_')} gm4_modules"

gm4/plugins/output.py

@@ -6,7 +6,8 @@
 import requests
 import shutil
 import logging
-from gm4.utils import run
+from gm4.utils import run, Version, NoneAttribute
+import gm4.plugins.manifest # for ManifestCacheModel; a runtime circular dependency
 
 parent_logger = logging.getLogger("gm4.output")
 
@@ -19,9 +20,11 @@
 
 class ModrinthConfig(PluginOptions):
 	project_id: Optional[str]
+	minecraft: list[str] = SUPPORTED_GAME_VERSIONS
 
 class SmithedConfig(PluginOptions):
 	pack_id: Optional[str]
+	minecraft: list[str] = SUPPORTED_GAME_VERSIONS
 
 class PMCConfig(PluginOptions):
 	uid: Optional[int]
@@ -55,7 +58,10 @@ def release(ctx: Context):
 	"""
 	version_dir = os.getenv("VERSION", "1.20")
 	release_dir = Path("release") / version_dir
-	file_name = f"{ctx.project_id}_{version_dir.replace('.', '_')}.zip"
+
+	corrected_project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id
+
+	file_name = f"{corrected_project_id}_{version_dir.replace('.', '_')}.zip"
 
 	yield # wait for exit phase, after other plugins cleanup
 
@@ -70,37 +76,36 @@ def release(ctx: Context):
 	pack_icon_dir = generated_dir / "pack_icons"
 	os.makedirs(pack_icon_dir, exist_ok=True)
 	if "pack.png" in ctx.data.extra:
-		ctx.data.extra["pack.png"].dump(pack_icon_dir, f"{ctx.project_id}.png")
+		ctx.data.extra["pack.png"].dump(pack_icon_dir, f"{corrected_project_id}.png")
 
 	smithed_readme_dir = generated_dir / "smithed_readmes"
 	os.makedirs(smithed_readme_dir, exist_ok=True)
 	if "smithed_readme" in ctx.meta:
-		ctx.meta['smithed_readme'].dump(smithed_readme_dir, f"{ctx.project_id}.md")
+		ctx.meta['smithed_readme'].dump(smithed_readme_dir, f"{corrected_project_id}.md")
 
 	# publish to download platforms
 	publish_modrinth(ctx, release_dir, file_name)
-	publish_smithed(ctx, release_dir, file_name)
+	publish_smithed(ctx, file_name)
+
 
 def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
 	'''Attempts to publish pack to modrinth'''
-	modrinth = ctx.meta.get("modrinth", None)
+	opts = ctx.validate("modrinth", ModrinthConfig)
 	auth_token = os.getenv(MODRINTH_AUTH_KEY, None)
 	logger = parent_logger.getChild(f"modrinth.{ctx.project_id}")
-	if modrinth and auth_token:
-		modrinth_id = modrinth["project_id"]
-		
+	if opts.project_id and auth_token:
 		# update page description
-		res = requests.get(f"{MODRINTH_API}/project/{modrinth_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
+		res = requests.get(f"{MODRINTH_API}/project/{opts.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
 		if not (200 <= res.status_code < 300):
 			if res.status_code == 404:
-				logger.warning(f"Cannot edit description of modrinth project {modrinth_id} as it doesn't exist.")
+				logger.warning(f"Cannot edit description of modrinth project {opts.project_id} as it doesn't exist.")
 			else:
 				logger.warning(f"Failed to get project: {res.status_code} {res.text}")
 			return
 		existing_readme = res.json()["body"]
 		if existing_readme != (d:=ctx.meta['modrinth_readme'].text):
 			logger.debug("Readme and modrinth-page content differ. Updating webpage body")
-			res = requests.patch(f"{MODRINTH_API}/project/{modrinth_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={"body": d})
+			res = requests.patch(f"{MODRINTH_API}/project/{opts.project_id}", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT}, json={"body": d})
 			if not (200 <= res.status_code < 300):
 				logger.warning(f"Failed to update description: {res.status_code} {res.text}")
 			logger.info(f"Successfully updated description of {ctx.project_name}")
@@ -112,16 +117,16 @@ def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
 				logger.warning("Full version number not available in ctx.meta. Skipping publishing")
 				return
 
-			res = requests.get(f"{MODRINTH_API}/project/{modrinth_id}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
+			res = requests.get(f"{MODRINTH_API}/project/{opts.project_id}/version", headers={'Authorization': auth_token, 'User-Agent': USER_AGENT})
 			if not (200 <= res.status_code < 300):
 				if res.status_code == 404:
-					logger.warning(f"Cannot publish to modrinth project {modrinth_id} as it doesn't exist.")
+					logger.warning(f"Cannot publish to modrinth project {opts.project_id} as it doesn't exist.")
 				else:
 					logger.warning(f"Failed to get project versions: {res.status_code} {res.text}")
 				return
 			project_data = res.json()
 			matching_version = next((v for v in project_data if v["version_number"] == str(version)), None)
-			game_versions = modrinth.get("minecraft", SUPPORTED_GAME_VERSIONS)
+			game_versions = opts.minecraft
 			if matching_version is not None: # patch version already exists
 				# update mc versions if necessary
 				if not set(matching_version["game_versions"]) == set(game_versions):
@@ -148,7 +153,7 @@ def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
 					"version_type": "release",
 					"loaders": ["datapack"],
 					"featured": False,
-					"project_id": modrinth_id,
+					"project_id": opts.project_id,
 					"file_parts": [file_name],
 				}),
 				file_name: file_bytes,
@@ -158,54 +163,58 @@ def publish_modrinth(ctx: Context, release_dir: Path, file_name: str):
 				return
 			logger.info(f"Successfully published {res.json()['name']}")
 
-def publish_smithed(ctx: Context, release_dir: Path, file_name: str):
+def publish_smithed(ctx: Context, file_name: str):
 	"""Attempts to publish pack to smithed"""
-	smithed = ctx.meta.get("smithed", None)
+	opts = ctx.validate("smithed", SmithedConfig)
 	auth_token = os.getenv(SMITHED_AUTH_KEY, None)
 	logger = parent_logger.getChild(f"smithed.{ctx.project_id}")
 	mc_version_dir = os.getenv("VERSION", "1.20")
-	if smithed and auth_token and ctx.project_version:
-		version = ctx.cache["gm4_manifest"].json["modules"].get(ctx.project_id, {}).get("version", None)
-		smithed_id = smithed["pack_id"]
+	manifest = gm4.plugins.manifest.ManifestCacheModel.parse_obj(ctx.cache["gm4_manifest"].json)
+	project_id = stem if (stem:=ctx.directory.stem).startswith("lib") else ctx.project_id
+
+	if opts.pack_id and auth_token:
+		version = (manifest.modules|manifest.libraries).get(project_id, NoneAttribute()).version or ""
 
 		# get project data and existing versions
-		res = requests.get(f"{SMITHED_API}/packs/{smithed_id}")
+		res = requests.get(f"{SMITHED_API}/packs/{opts.pack_id}")
 		if not (200 <= res.status_code < 300):
 			if res.status_code == 404:
-				logger.warning(f"Cannot publish to smithed project {smithed_id} as it doesn't exist.")
+				logger.warning(f"Cannot publish to smithed project {opts.pack_id} as it doesn't exist.")
 			else:
 				logger.warning(f"Failed to get project: {res.status_code} {res.text}")
 			return
 
 		project_data = res.json()
-		
+
 		# update description and pack image
 			# ensures they point to the most up-to-date mc version branch
-		project_display = project_data["display"]
-		current_icon = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/pack_icons/{ctx.project_id}.png"
-		current_readme = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/smithed_readmes/{ctx.project_id}.md"
-
-		if project_display["icon"] != current_icon or project_display["webPage"] != current_readme:
-			logger.debug("Pack Icon or Readme hyperlink is incorrect. Updating project")
-			res = requests.patch(f"{SMITHED_API}/packs/{smithed_id}", params={'token': auth_token},
-				json={"data": {
-						"display": {
-							"icon": current_icon,
-							"webPage": current_readme,
-						},
-				}})
-			if not (200 <= res.status_code < 300):
-				logger.warning(f"Failed to update descripion: {res.status_code} {res.text}")
-			logger.info(f"{ctx.project_name} {res.text}")
-
 		project_versions = project_data["versions"]
+		newest_version = sorted([Version(v["name"]) for v in project_versions])[-1]
+		if Version(version) > newest_version: # only update the description if we're not patching an old version
+			project_display = project_data["display"]
+			current_icon = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/pack_icons/{project_id}.png"
+			current_readme = f"https://raw.githubusercontent.com/Gamemode4Dev/GM4_Datapacks/release/{mc_version_dir}/generated/smithed_readmes/{project_id}.md"
+
+			if project_display["icon"] != current_icon or project_display["webPage"] != current_readme:
+				logger.debug("Pack Icon or Readme hyperlink is incorrect. Updating project")
+				res = requests.patch(f"{SMITHED_API}/packs/{opts.pack_id}", params={'token': auth_token},
+					json={"data": {
+							"display": {
+								"icon": current_icon,
+								"webPage": current_readme,
+							},
+					}})
+				if not (200 <= res.status_code < 300):
+					logger.warning(f"Failed to update descripion: {res.status_code} {res.text}")
+				logger.info(f"{ctx.project_name} {res.text}")
+
 		matching_version = next((v for v in project_versions if v["name"] == str(version)), None)
-		game_versions = smithed.get("minecraft", SUPPORTED_GAME_VERSIONS)
+		game_versions = opts.minecraft
 		if matching_version is not None: # patch version already exists
 			# update MC version if necessary
 			if not set(matching_version["supports"]) == set(game_versions):
 				logger.debug("Additional MC version support has been added to an existing patch version. Updating existing smithed version data")
-				res = requests.patch(f"{SMITHED_API}/packs/{smithed_id}/versions/{matching_version['name']}", params={'token': auth_token}, json={
+				res = requests.patch(f"{SMITHED_API}/packs/{opts.pack_id}/versions/{matching_version['name']}", params={'token': auth_token}, json={
 					"data": {
 						"supports": game_versions
 					}
@@ -214,17 +223,27 @@ def publish_smithed(ctx: Context, release_dir: Path, file_name: str):
 					logger.warning(f"Failed to patch project versions: {res.status_code} {res.text}")
 			return
 
-		# remove other existing versions for that mc version
-		mc_version_matching_version = (v["name"] for v in project_versions if set(v['supports']) & set(game_versions))
-		for v in mc_version_matching_version:
-			res = requests.delete(f"{SMITHED_API}/packs/{smithed_id}/versions/{v}", params={'token': auth_token})
+		# permalink previous version (in that MC version) to the git history
+		commit_hash = run("cd release && git log -1 --format=%H")
+		matching_mc_versions = sorted((Version(v["name"]) for v in project_versions if set(v['supports']) & set(game_versions)))
+		prior_version_in_mc_version = matching_mc_versions[-1] if len(matching_mc_versions) > 0 else None # newest version number, with any MC overlap
+		prior_url: str = next((v["downloads"]["datapack"] for v in project_versions if Version(v["name"]) == prior_version_in_mc_version), "")
+		if "https://github.com/Gamemode4Dev/GM4_Datapacks/blob/" not in prior_url and prior_version_in_mc_version:
+			res = requests.patch(f"{SMITHED_API}/packs/{opts.pack_id}/versions/{prior_version_in_mc_version}", params={'token': auth_token}, json={
+				"data":{
+					"downloads": {
+					"datapack": f"https://github.com/Gamemode4Dev/GM4_Datapacks/blob/{commit_hash}/{mc_version_dir}/{file_name}?raw=true",
+					"resourcepack": ""
+					}
+				}
+			})
 			if not (200 <= res.status_code < 300):
-				logger.warning(f"Failed to delete {ctx.project_name} version {v}: {res.status_code} {res.text}")
+				logger.warning(f"Failed to permalink {project_id} version {prior_version_in_mc_version}: {res.status_code} {res.text}")
 			else:
-				logger.info(f"{ctx.project_name} {res.text}")
+				logger.info(f"Permalinked {project_id} {prior_version_in_mc_version} to git history: {res.text}")
 
 		# post new version
-		res = requests.post(f"{SMITHED_API}/packs/{smithed_id}/versions",
+		res = requests.post(f"{SMITHED_API}/packs/{opts.pack_id}/versions",
 		    params={'token': auth_token, 'version': version},
 			json={"data":{
 				"downloads":{