diff --git a/__init__.py b/__init__.py index 9d636b2..f04289b 100644 --- a/__init__.py +++ b/__init__.py @@ -3,10 +3,15 @@ import glob import importlib import os +import pathlib import site import sys import typing +from PyQt6.QtCore import qWarning + +from mobase import IPlugin + from .basic_game import BasicGame from .basic_game_ini import BasicIniGame @@ -18,7 +23,7 @@ def createPlugins(): # List of game class from python: - game_plugins: typing.List[BasicGame] = [] + game_plugins: typing.List[IPlugin] = [] # We are going to list all game plugins: curpath = os.path.abspath(os.path.dirname(__file__)) @@ -58,5 +63,25 @@ def createPlugins(): "Failed to instantiate {}: {}".format(name, e), file=sys.stderr, ) + for path in pathlib.Path(escaped_games_path).rglob("plugins/__init__.py"): + module_path = "." + os.path.relpath(path.parent, curpath).replace(os.sep, ".") + try: + module = importlib.import_module(module_path, __package__) + if hasattr(module, "createPlugins") and callable(module.createPlugins): + try: + plugins: typing.Any = module.createPlugins() + for item in plugins: + if isinstance(item, IPlugin): + game_plugins.append(item) + except TypeError: + pass + if hasattr(module, "createPlugin") and callable(module.createPlugin): + plugin = module.createPlugin() + if isinstance(plugin, IPlugin): + game_plugins.append(plugin) + except ImportError as e: + qWarning(f"Error importing module {module_path}: {e}") + except Exception as e: + qWarning(f"Error calling function createPlugin(s) in {module_path}: {e}") return game_plugins diff --git a/basic_game.py b/basic_game.py index 09fca10..b3d8073 100644 --- a/basic_game.py +++ b/basic_game.py @@ -640,7 +640,7 @@ def setGamePath(self, path: Path | str) -> None: self._mappings.steamAPPId.set_value(steamid) for gogid, gogpath in BasicGame.gog_games.items(): if gogpath == path: - self._mappings.steamAPPId.set_value(gogid) + self._mappings.gogAPPId.set_value(gogid) for originid, originpath in BasicGame.origin_games.items(): if originpath == path: self._mappings.originManifestIds.set_value(originid) diff --git a/games/baldursgate3/__init__.py b/games/baldursgate3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/games/baldursgate3/bg3_data_checker.py b/games/baldursgate3/bg3_data_checker.py new file mode 100644 index 0000000..695d21c --- /dev/null +++ b/games/baldursgate3/bg3_data_checker.py @@ -0,0 +1,58 @@ +from pathlib import Path + +import mobase + +from ...basic_features import BasicModDataChecker, GlobPatterns, utils +from . import bg3_utils + + +class BG3ModDataChecker(BasicModDataChecker): + def __init__(self): + super().__init__( + GlobPatterns( + valid=[ + "*.pak", + str(Path("Mods") / "*.pak"), # standard mods + "bin", # native mods / Script Extender + "Script Extender", # mods which are configured via jsons in this folder + "Data", # loose file mods + ] + + [str(Path("*") / f) for f in bg3_utils.loose_file_folders], + move={ + "Root/": "", # root builder not needed + "*.dll": "bin/", + "ScriptExtenderSettings.json": "bin/", + } + | {f: "Data/" for f in bg3_utils.loose_file_folders}, + delete=["info.json", "*.txt"], + ) + ) + + def dataLooksValid( + self, filetree: mobase.IFileTree + ) -> mobase.ModDataChecker.CheckReturn: + status = mobase.ModDataChecker.INVALID + rp = self._regex_patterns + for entry in filetree: + name = entry.name().casefold() + if rp.unfold.match(name): + if utils.is_directory(entry): + status = self.dataLooksValid(entry) + else: + status = mobase.ModDataChecker.INVALID + break + elif rp.valid.match(name): + if status is mobase.ModDataChecker.INVALID: + status = mobase.ModDataChecker.VALID + elif isinstance(entry, mobase.IFileTree): + status = ( + mobase.ModDataChecker.VALID + if all(rp.valid.match(e.pathFrom(filetree)) for e in entry) + else mobase.ModDataChecker.INVALID + ) + elif rp.delete.match(name) or rp.move_match(name) is not None: + status = mobase.ModDataChecker.FIXABLE + else: + status = mobase.ModDataChecker.INVALID + break + return status diff --git a/games/baldursgate3/bg3_data_content.py b/games/baldursgate3/bg3_data_content.py new file mode 100644 index 0000000..c378520 --- /dev/null +++ b/games/baldursgate3/bg3_data_content.py @@ -0,0 +1,54 @@ +from enum import IntEnum, auto + +import mobase + +from . import bg3_utils + + +class Content(IntEnum): + PAK = auto() + WORKSPACE = auto() + NATIVE = auto() + LOOSE_FILES = auto() + SE_FILES = auto() + + +class BG3DataContent(mobase.ModDataContent): + BG3_CONTENTS: list[tuple[Content, str, str, bool] | tuple[Content, str, str]] = [ + (Content.WORKSPACE, "Mod workspace", ":/MO/gui/content/script"), + (Content.PAK, "Pak", ":/MO/gui/content/bsa"), + (Content.LOOSE_FILES, "Loose file override mod", ":/MO/gui/content/texture"), + (Content.SE_FILES, "Script Extender Files", ":/MO/gui/content/inifile"), + (Content.NATIVE, "Native DLL mod", ":/MO/gui/content/plugin"), + ] + + def getAllContents(self) -> list[mobase.ModDataContent.Content]: + return [ + mobase.ModDataContent.Content(id, name, icon, *filter_only) + for id, name, icon, *filter_only in self.BG3_CONTENTS + ] + + def getContentsFor(self, filetree: mobase.IFileTree) -> list[int]: + contents: set[int] = set() + for entry in filetree: + if isinstance(entry, mobase.IFileTree): + match entry.name(): + case "Script Extender": + contents.add(Content.SE_FILES) + case "Data": + contents.add(Content.LOOSE_FILES) + case "Mods": + for e in entry: + if e.name().endswith(".pak"): + contents.add(Content.PAK) + break + case "bin": + contents.add(Content.NATIVE) + case _: + for e in entry: + if e.name() in bg3_utils.loose_file_folders: + contents.add(Content.WORKSPACE) + break + elif entry.name().endswith(".pak"): + contents.add(Content.PAK) + return list(contents) diff --git a/games/baldursgate3/bg3_file_mapper.py b/games/baldursgate3/bg3_file_mapper.py new file mode 100644 index 0000000..ba0b987 --- /dev/null +++ b/games/baldursgate3/bg3_file_mapper.py @@ -0,0 +1,104 @@ +import functools +import json +import os +from pathlib import Path +from typing import Callable, Optional + +import yaml +from PyQt6.QtCore import QDir, qDebug, qInfo, qWarning +from PyQt6.QtWidgets import QApplication + +import mobase + +from . import bg3_utils + + +class BG3FileMapper(mobase.IPluginFileMapper): + current_mappings: list[mobase.Mapping] = [] + + def __init__(self, utils: bg3_utils.BG3Utils, doc_dir: Callable[[], QDir]): + super().__init__() + self._utils = utils + self.doc_dir = doc_dir + + @functools.cached_property + def doc_path(self): + return Path(self.doc_dir().path()) + + def mappings(self) -> list[mobase.Mapping]: + qInfo("creating custom bg3 mappings") + self.current_mappings.clear() + active_mods = self._utils.active_mods() + doc_dir = Path(self.doc_dir().path()) + progress = self._utils.create_progress_window( + "Mapping files to documents folder", len(active_mods) + 1 + ) + docs_path_mods = doc_dir / "Mods" + docs_path_se = doc_dir / "Script Extender" + for mod in active_mods: + modpath = Path(mod.absolutePath()) + self.map_files(modpath, dest=docs_path_mods, pattern="*.pak", rel=False) + self.map_files(modpath / "Script Extender", dest=docs_path_se) + progress.setValue(progress.value() + 1) + QApplication.processEvents() + if progress.wasCanceled(): + qWarning("mapping canceled by user") + return self.current_mappings + self.map_files(self._utils.overwrite_path) + self.create_mapping( + self._utils.modsettings_path, + doc_dir / "PlayerProfiles" / "Public" / self._utils.modsettings_path.name, + ) + progress.setValue(len(active_mods) + 1) + QApplication.processEvents() + progress.close() + return self.current_mappings + + def map_files( + self, + path: Path, + dest: Optional[Path] = None, + pattern: str = "*", + rel: bool = True, + ): + dest = dest if dest else self.doc_path + dest_func: Callable[[Path], str] = ( + (lambda f: os.path.relpath(f, path)) if rel else lambda f: f.name + ) + found_jsons: set[Path] = set() + for file in list(path.rglob(pattern)): + if self._utils.convert_yamls_to_json and ( + file.name.endswith(".yaml") or file.name.endswith(".yml") + ): + converted_path = file.parent / file.name.replace( + ".yaml", ".json" + ).replace(".yml", ".json") + try: + if not converted_path.exists() or os.path.getmtime( + file + ) > os.path.getmtime(converted_path): + with open(file, "r") as yaml_file: + with open(converted_path, "w") as json_file: + json.dump( + yaml.safe_load(yaml_file), json_file, indent=2 + ) + qDebug(f"Converted {file} to JSON") + found_jsons.add(converted_path) + except OSError as e: + qWarning(f"Error accessing file {converted_path}: {e}") + elif file.name.endswith(".json"): + found_jsons.add(file) + else: + self.create_mapping(file, dest / dest_func(file)) + for file in found_jsons: + self.create_mapping(file, dest / dest_func(file)) + + def create_mapping(self, file: Path, dest: Path): + self.current_mappings.append( + mobase.Mapping( + source=str(file), + destination=str(dest), + is_directory=file.is_dir(), + create_target=True, + ) + ) diff --git a/games/baldursgate3/bg3_utils.py b/games/baldursgate3/bg3_utils.py new file mode 100644 index 0000000..a61890c --- /dev/null +++ b/games/baldursgate3/bg3_utils.py @@ -0,0 +1,242 @@ +import functools +import shutil +import typing +from pathlib import Path + +from PyQt6.QtCore import ( + QCoreApplication, + QDir, + QEventLoop, + QRunnable, + Qt, + QThread, + QThreadPool, + qInfo, + qWarning, +) +from PyQt6.QtWidgets import QApplication, QMainWindow, QProgressDialog + +import mobase + +loose_file_folders = { + "Public", + "Mods", + "Generated", + "Localization", + "ScriptExtender", +} + + +class BG3Utils: + _mod_settings_xml_start = """ + + + + + + + + + + + + + + + """ + _mod_settings_xml_end = """ + + + + + + """ + + def __init__(self, name: str): + self.main_window = None + self._name = name + from . import lslib_retriever, pak_parser + + self.lslib_retriever = lslib_retriever.LSLibRetriever(self) + self._pak_parser = pak_parser.BG3PakParser(self) + + def init(self, organizer: mobase.IOrganizer): + self._organizer = organizer + + @functools.cached_property + def autobuild_paks(self): + return bool(self.get_setting("autobuild_paks")) + + @functools.cached_property + def extract_full_package(self): + return bool(self.get_setting("extract_full_package")) + + @functools.cached_property + def remove_extracted_metadata(self): + return bool(self.get_setting("remove_extracted_metadata")) + + @functools.cached_property + def force_load_dlls(self): + return bool(self.get_setting("force_load_dlls")) + + @functools.cached_property + def log_diff(self): + return bool(self.get_setting("log_diff")) + + @functools.cached_property + def convert_yamls_to_json(self): + return bool(self.get_setting("convert_yamls_to_json")) + + @functools.cached_property + def log_dir(self): + return Path(self._organizer.basePath()) / "logs" + + @functools.cached_property + def modsettings_backup(self): + return self.plugin_data_path / "temp" / "modsettings.lsx" + + @functools.cached_property + def modsettings_path(self): + return Path(self._organizer.profilePath()) / "modsettings.lsx" + + @functools.cached_property + def plugin_data_path(self) -> Path: + """Gets the path to the data folder for the current plugin.""" + return Path(self._organizer.pluginDataPath(), self._name).absolute() + + @functools.cached_property + def tools_dir(self): + return self.plugin_data_path / "tools" + + @functools.cached_property + def overwrite_path(self): + return Path(self._organizer.overwritePath()) + + def active_mods(self) -> list[mobase.IModInterface]: + modlist = self._organizer.modList() + return [ + modlist.getMod(mod_name) + for mod_name in filter( + lambda mod: modlist.state(mod) & mobase.ModState.ACTIVE, + modlist.allModsByProfilePriority(), + ) + ] + + def _set_setting(self, key: str, value: mobase.MoVariant): + self._organizer.setPluginSetting(self._name, key, value) + + def get_setting(self, key: str) -> mobase.MoVariant: + return self._organizer.pluginSetting(self._name, key) + + def tr(self, trstr: str) -> str: + return QCoreApplication.translate(self._name, trstr) + + def create_progress_window( + self, title: str, max_progress: int, msg: str = "", cancelable: bool = True + ) -> QProgressDialog: + progress = QProgressDialog( + self.tr(msg if msg else title), + self.tr("Cancel") if cancelable else None, + 0, + max_progress, + self.main_window, + ) + progress.setWindowTitle(self.tr(f"BG3 Plugin: {title}")) + progress.setWindowModality(Qt.WindowModality.ApplicationModal) + progress.show() + return progress + + def on_user_interface_initialized(self, window: QMainWindow) -> None: + self.main_window = window + + def on_settings_changed( + self, + plugin_name: str, + setting: str, + old: mobase.MoVariant, + new: mobase.MoVariant, + ) -> None: + if self._name != plugin_name: + return + if setting in { + "extract_full_package", + "autobuild_paks", + "remove_extracted_metadata", + "force_load_dlls", + "log_diff", + "convert_yamls_to_json", + } and hasattr(self, setting): + delattr(self, setting) + + def construct_modsettings_xml( + self, + exec_path: str = "", + working_dir: typing.Optional[QDir] = None, + args: str = "", + force_reparse_metadata: bool = False, + ) -> bool: + if ( + "bin/bg3" not in exec_path + or not self.lslib_retriever.download_lslib_if_missing() + ): + return True + active_mods = self.active_mods() + progress = self.create_progress_window( + "Generating modsettings.xml", len(active_mods) + ) + threadpool = QThreadPool.globalInstance() + if threadpool is None: + return False + metadata: dict[str, str] = {} + + def retrieve_mod_metadata_in_new_thread(mod: mobase.IModInterface): + return lambda: metadata.update( + self._pak_parser.get_metadata_for_files_in_mod( + mod, force_reparse_metadata + ) + ) + + for mod in active_mods: + if progress.wasCanceled(): + qWarning("processing canceled by user") + return False + threadpool.start(QRunnable.create(retrieve_mod_metadata_in_new_thread(mod))) + count = 0 + num_active_mods = len(active_mods) + total_intervals_to_wait = (num_active_mods * 2) + 20 + while len(metadata.keys()) < num_active_mods: + progress.setValue(len(metadata.keys())) + QApplication.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, 100) + count += 1 + if count == total_intervals_to_wait or progress.wasCanceled(): + remaining_mods = {mod.name() for mod in active_mods} - metadata.keys() + qWarning(f"processing did not finish in time for: {remaining_mods}") + progress.close() + break + QThread.msleep(100) + progress.setValue(num_active_mods) + QApplication.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, 100) + progress.close() + qInfo(f"writing mod load order to {self.modsettings_path}") + self.modsettings_path.parent.mkdir(parents=True, exist_ok=True) + self.modsettings_path.write_text( + ( + self._mod_settings_xml_start + + "".join( + metadata[mod.name()] + for mod in active_mods + if mod.name() in metadata + ) + + self._mod_settings_xml_end + ) + ) + qInfo( + f"backing up generated file {self.modsettings_path} to {self.modsettings_backup}, " + f"check the backup after the executable runs for differences with the file used by the game if you encounter issues" + ) + shutil.copy(self.modsettings_path, self.modsettings_backup) + return True + + def on_mod_installed(self, mod: mobase.IModInterface) -> None: + if self.lslib_retriever.download_lslib_if_missing(): + self._pak_parser.get_metadata_for_files_in_mod(mod, True) diff --git a/games/baldursgate3/lslib_retriever.py b/games/baldursgate3/lslib_retriever.py new file mode 100644 index 0000000..9a7b7af --- /dev/null +++ b/games/baldursgate3/lslib_retriever.py @@ -0,0 +1,157 @@ +import json +import shutil +import traceback +import urllib.request +import zipfile +from functools import cached_property + +from PyQt6.QtCore import qDebug, qWarning +from PyQt6.QtWidgets import QApplication, QMessageBox + +from . import bg3_utils + + +class LSLibRetriever: + def __init__(self, utils: bg3_utils.BG3Utils): + self._utils = utils + + @cached_property + def _needed_lslib_files(self): + return { + self._utils.tools_dir / x + for x in { + "CommandLineArgumentsParser.dll", + "Divine.dll", + "Divine.dll.config", + "Divine.exe", + "Divine.runtimeconfig.json", + "LSLib.dll", + "LSLibNative.dll", + "LZ4.dll", + "System.IO.Hashing.dll", + "ZstdSharp.dll", + } + } + + def download_lslib_if_missing(self, force: bool = False) -> bool: + if not force and all(x.exists() for x in self._needed_lslib_files): + return True + try: + self._utils.tools_dir.mkdir(exist_ok=True, parents=True) + downloaded = False + + def reporthook(block_num: int, block_size: int, total_size: int) -> None: + if total_size > 0: + progress.setValue( + min(int(block_num * block_size * 100 / total_size), 100) + ) + QApplication.processEvents() + + with urllib.request.urlopen( + "https://api.github.com/repos/Norbyte/lslib/releases/latest" + ) as response: + assets = json.loads(response.read().decode("utf-8"))["assets"][0] + zip_path = self._utils.tools_dir / assets["name"] + if not zip_path.exists(): + old_archives = list(self._utils.tools_dir.glob("*.zip")) + msg_box = QMessageBox(self._utils.main_window) + msg_box.setWindowTitle( + self._utils.tr("Baldur's Gate 3 Plugin - Missing dependencies") + ) + if old_archives: + msg_box.setText(self._utils.tr("LSLib update available.")) + else: + msg_box.setText( + self._utils.tr( + "LSLib tools are missing.\nThese are necessary for the plugin to create the load order file for BG3." + ) + ) + msg_box.addButton( + self._utils.tr("Download"), + QMessageBox.ButtonRole.DestructiveRole, + ) + exit_btn = msg_box.addButton( + self._utils.tr("Exit"), QMessageBox.ButtonRole.ActionRole + ) + msg_box.setIcon(QMessageBox.Icon.Warning) + msg_box.exec() + + if msg_box.clickedButton() == exit_btn: + if not old_archives: + err = QMessageBox(self._utils.main_window) + err.setIcon(QMessageBox.Icon.Critical) + err.setText( + "LSLib tools are required for the proper generation of the modsettings.xml file, file will not be generated" + ) + return False + else: + progress = self._utils.create_progress_window( + "Downloading LSLib", 100, cancelable=False + ) + urllib.request.urlretrieve( + assets["browser_download_url"], str(zip_path), reporthook + ) + progress.close() + downloaded = True + for archive in old_archives: + archive.unlink() + old_archives = [] + else: + old_archives = [] + new_msg = QMessageBox(self._utils.main_window) + new_msg.setIcon(QMessageBox.Icon.Information) + new_msg.setText( + self._utils.tr("Latest version of LSLib already downloaded!") + ) + + except Exception as e: + qDebug(f"Download failed: {e}") + err = QMessageBox(self._utils.main_window) + err.setIcon(QMessageBox.Icon.Critical) + err.setText( + self._utils.tr( + f"Failed to download LSLib tools:\n{traceback.format_exc()}" + ) + ) + err.exec() + return False + try: + if old_archives: + zip_path = sorted(old_archives)[-1] + if old_archives or not downloaded: + dialog_message = "Ensuring all necessary LSLib files have been extracted from archive..." + win_title = "Verifying LSLib files" + else: + dialog_message = "Extracting/Updating LSLib files..." + win_title = "Extracting LSLib" + x_progress = self._utils.create_progress_window( + win_title, len(self._needed_lslib_files), msg=dialog_message + ) + with zipfile.ZipFile(zip_path, "r") as zip_ref: + for file in self._needed_lslib_files: + if downloaded or not file.exists(): + shutil.move( + zip_ref.extract( + f"Packed/Tools/{file.name}", self._utils.tools_dir + ), + file, + ) + x_progress.setValue(x_progress.value() + 1) + QApplication.processEvents() + if x_progress.wasCanceled(): + qWarning("processing canceled by user") + return False + x_progress.close() + shutil.rmtree(self._utils.tools_dir / "Packed", ignore_errors=True) + except Exception as e: + qDebug(f"Extraction failed: {e}") + err = QMessageBox(self._utils.main_window) + err.setIcon(QMessageBox.Icon.Critical) + err.setText( + self._utils.tr( + f"Failed to extract LSLib tools:\n{traceback.format_exc()}" + ) + ) + err.exec() + return False + return True diff --git a/games/baldursgate3/pak_parser.py b/games/baldursgate3/pak_parser.py new file mode 100644 index 0000000..89debd5 --- /dev/null +++ b/games/baldursgate3/pak_parser.py @@ -0,0 +1,290 @@ +import configparser +import hashlib +import itertools +import os +import re +import shutil +import subprocess +import traceback +from functools import cached_property +from pathlib import Path +from typing import Callable +from xml.etree import ElementTree +from xml.etree.ElementTree import Element + +from PyQt6.QtCore import ( + qDebug, + qInfo, + qWarning, +) + +import mobase + +from . import bg3_utils + + +class BG3PakParser: + def __init__(self, utils: bg3_utils.BG3Utils): + self._utils = utils + + _mod_cache: dict[Path, bool] = {} + _types = { + "Folder": "", + "MD5": "", + "Name": "", + "PublishHandle": "0", + "UUID": "", + "Version64": "0", + } + + @cached_property + def _divine_command(self): + return f"{self._utils.tools_dir / 'Divine.exe'} -g bg3 -l info" + + @cached_property + def _folder_pattern(self): + return re.compile("Data|Script Extender|bin|Mods") + + def get_metadata_for_files_in_mod( + self, mod: mobase.IModInterface, force_reparse_metadata: bool + ): + return { + mod.name(): "".join( + [ + self._get_metadata_for_file(mod, file, force_reparse_metadata) + for file in sorted( + list(Path(mod.absolutePath()).rglob("*.pak")) + + ( + [ + f + for f in Path(mod.absolutePath()).glob("*") + if f.is_dir() + ] + if self._utils.autobuild_paks + else [] + ) + ) + ] + ) + } + + def _get_metadata_for_file( + self, + mod: mobase.IModInterface, + file: Path, + force_reparse_metadata: bool, + ) -> str: + meta_ini = Path(mod.absolutePath()) / "meta.ini" + config = configparser.ConfigParser() + config.read(meta_ini, encoding="utf-8") + try: + if file.name.endswith("pak"): + meta_file = ( + self._utils.plugin_data_path + / "temp" + / "extracted_metadata" + / f"{file.name[: int(len(file.name) / 2)]}-{hashlib.md5(str(file).encode(), usedforsecurity=False).hexdigest()[:5]}.lsx" + ) + try: + if ( + not force_reparse_metadata + and config.has_section(file.name) + and ( + "override" in config[file.name].keys() + or "Folder" in config[file.name].keys() + ) + ): + return get_module_short_desc(config, file) + meta_file.parent.mkdir(parents=True, exist_ok=True) + meta_file.unlink(missing_ok=True) + out_dir = ( + str(meta_file)[:-4] if self._utils.extract_full_package else "" + ) + can_continue = True + if self.run_divine( + f'{"extract-package" if self._utils.extract_full_package else "extract-single-file -f meta.lsx"} -d "{meta_file if not self._utils.extract_full_package else out_dir}"', + file, + ).returncode: + can_continue = False + if can_continue and self._utils.extract_full_package: + qDebug(f"archive {file} extracted to {out_dir}") + if self.run_divine( + f'convert-resources -d "{out_dir}" -i lsf -o lsx -x "*.lsf"', + out_dir, + ).returncode: + qDebug( + f"failed to convert lsf files in {out_dir} to readable lsx" + ) + extracted_meta_files = list(Path(out_dir).rglob("meta.lsx")) + if len(extracted_meta_files) == 0: + qInfo( + f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod" + ) + can_continue = False + else: + shutil.copyfile( + extracted_meta_files[0], + meta_file, + ) + elif can_continue and not meta_file.exists(): + qInfo( + f"No meta.lsx files found in {file.name}, {file.name} determined to be an override mod" + ) + can_continue = False + return self.metadata_to_ini( + config, file, mod, meta_ini, can_continue, lambda: meta_file + ) + finally: + if self._utils.remove_extracted_metadata: + meta_file.unlink(missing_ok=True) + if self._utils.extract_full_package: + Path(str(meta_file)[:-4]).unlink(missing_ok=True) + elif file.is_dir() and self._folder_pattern.search(file.name): + # qDebug(f"directory is not packable: {file}") + return "" + elif next( + itertools.chain( + file.glob(f"{folder}/*") for folder in bg3_utils.loose_file_folders + ), + False, + ): + qInfo(f"packable dir: {file}") + if (file.parent / f"{file.name}.pak").exists() or ( + file.parent / "Mods" / f"{file.name}.pak" + ).exists(): + qInfo( + f"pak with same name as packable dir exists in mod directory. not packing dir {file}" + ) + return "" + pak_path = self._utils.overwrite_path / f"Mods/{file.name}.pak" + build_pak = True + if pak_path.exists(): + pak_creation_time = os.path.getmtime(pak_path) + for root, _, files in os.walk(file): + for f in files: + file_path = os.path.join(root, f) + try: + if os.path.getmtime(file_path) > pak_creation_time: + break + except OSError as e: + qDebug(f"Error accessing file {file_path}: {e}") + break + else: + build_pak = False + if build_pak: + pak_path.unlink(missing_ok=True) + if self.run_divine( + f'create-package -d "{pak_path}"', file + ).returncode: + return "" + meta_files = list(file.glob("Mods/*/meta.lsx")) + return self.metadata_to_ini( + config, + file, + mod, + meta_ini, + len(meta_files) > 0, + lambda: meta_files[0], + ) + else: + # qDebug(f"non packable dir, unlikely to be used by the game: {file}") + return "" + except Exception: + qWarning(traceback.format_exc()) + return "" + + def run_divine( + self, action: str, source: Path | str + ) -> subprocess.CompletedProcess[str]: + command = f'{self._divine_command} -a {action} -s "{source}"' + result = subprocess.run( + command, + creationflags=subprocess.CREATE_NO_WINDOW, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode: + qWarning( + f"{command.replace(str(Path.home()), '~', 1).replace(str(Path.home()), '$HOME')}" + f" returned stdout: {result.stdout}, stderr: {result.stderr}, code {result.returncode}" + ) + return result + + def get_attr_value(self, root: Element, attr_id: str) -> str: + default_val = self._types.get(attr_id) or "" + attr = root.find(f".//attribute[@id='{attr_id}']") + return default_val if attr is None else attr.get("value", default_val) + + def metadata_to_ini( + self, + config: configparser.ConfigParser, + file: Path, + mod: mobase.IModInterface, + meta_ini: Path, + condition: bool, + to_parse: Callable[[], Path], + ): + config[file.name] = {} + if condition: + root = ( + ElementTree.parse(to_parse()) + .getroot() + .find(".//node[@id='ModuleInfo']") + ) + if root is None: + qInfo(f"No ModuleInfo node found in meta.lsx for {mod.name()} ") + else: + section = config[file.name] + folder_name = self.get_attr_value(root, "Folder") + if file.is_dir(): + self._mod_cache[file] = ( + len(list(file.glob(f"*/{folder_name}/**"))) > 1 + or len( + list(file.glob("Public/Engine/Timeline/MaterialGroups/*")) + ) + > 0 + ) + elif file not in self._mod_cache: + # a mod which has a meta.lsx and is not an override mod meets at least one of three conditions: + # 1. it has files in Public/Engine/Timeline/MaterialGroups, or + # 2. it has files in Mods// other than the meta.lsx file, or + # 3. it has files in Public/ + result = self.run_divine( + f'list-package --use-regex -x "(/{folder_name}/(?!meta\\.lsx))|(Public/Engine/Timeline/MaterialGroups)"', + file, + ) + self._mod_cache[file] = ( + result.returncode == 0 and result.stdout.strip() != "" + ) + if self._mod_cache[file]: + for key in self._types: + section[key] = self.get_attr_value(root, key) + else: + qInfo(f"pak {file.name} determined to be an override mod") + section["override"] = "True" + section["Folder"] = folder_name + else: + config[file.name]["override"] = "True" + with open(meta_ini, "w+", encoding="utf-8") as f: + config.write(f) + return get_module_short_desc(config, file) + + +def get_module_short_desc(config: configparser.ConfigParser, file: Path) -> str: + return ( + "" + if not config.has_section(file.name) + or "override" in config[file.name].keys() + or "Name" not in config[file.name].keys() + else f""" + + + + + + + + """ + ) diff --git a/games/baldursgate3/plugins/__init__.py b/games/baldursgate3/plugins/__init__.py new file mode 100644 index 0000000..1b69faf --- /dev/null +++ b/games/baldursgate3/plugins/__init__.py @@ -0,0 +1,13 @@ +import mobase + +from .check_for_lslib_updates_plugin import BG3ToolCheckForLsLibUpdates +from .convert_jsons_to_yaml_plugin import BG3ToolConvertJsonsToYaml +from .reparse_pak_metadata_plugin import BG3ToolReparsePakMetadata + + +def createPlugins() -> list[mobase.IPluginTool]: + return [ + BG3ToolCheckForLsLibUpdates(), + BG3ToolReparsePakMetadata(), + BG3ToolConvertJsonsToYaml(), + ] diff --git a/games/baldursgate3/plugins/bg3_tool_plugin.py b/games/baldursgate3/plugins/bg3_tool_plugin.py new file mode 100644 index 0000000..d77c5fc --- /dev/null +++ b/games/baldursgate3/plugins/bg3_tool_plugin.py @@ -0,0 +1,47 @@ +from pathlib import Path + +from PyQt6.QtCore import QCoreApplication +from PyQt6.QtGui import QIcon + +import mobase + + +class BG3ToolPlugin(mobase.IPluginTool, mobase.IPlugin): + icon_file = desc = sub_name = "" + + def __init__(self): + mobase.IPluginTool.__init__(self) + mobase.IPlugin.__init__(self) + self._pluginName = self._displayName = "BG3 Tools" + self._pluginVersion = mobase.VersionInfo(1, 0, 0) + + def init(self, organizer: mobase.IOrganizer) -> bool: + self._organizer = organizer + return True + + def version(self): + return self._pluginVersion + + def author(self): + return "daescha" + + def name(self): + return f"{self._pluginName}: {self.sub_name}" + + def displayName(self): + return f"{self._displayName}/{self.sub_name}" + + def tooltip(self): + return self.description() + + def enabledByDefault(self): + return self._organizer.managedGame().name() == "Baldur's Gate 3 Plugin" + + def settings(self) -> list[mobase.PluginSetting]: + return [] + + def icon(self) -> QIcon: + return QIcon(str(Path(__file__).parent / "icons" / self.icon_file)) + + def description(self) -> str: + return QCoreApplication.translate(self._pluginName, self.desc) diff --git a/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py b/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py new file mode 100644 index 0000000..bc4df8c --- /dev/null +++ b/games/baldursgate3/plugins/check_for_lslib_updates_plugin.py @@ -0,0 +1,14 @@ +from .bg3_tool_plugin import BG3ToolPlugin + + +class BG3ToolCheckForLsLibUpdates(BG3ToolPlugin): + icon_file = "ui-update.ico" + sub_name = "Check For LsLib Updates" + desc = "Check to see if there has been a new release of LSLib and create download dialog if so." + + def display(self): + from ...game_baldursgate3 import BG3Game + + game_plugin = self._organizer.managedGame() + if isinstance(game_plugin, BG3Game): + game_plugin.utils.lslib_retriever.download_lslib_if_missing(True) diff --git a/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py b/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py new file mode 100644 index 0000000..58a51bb --- /dev/null +++ b/games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py @@ -0,0 +1,57 @@ +import json +import os +from pathlib import Path + +from PyQt6.QtCore import qInfo, qWarning +from PyQt6.QtWidgets import QApplication + +from .bg3_tool_plugin import BG3ToolPlugin + + +class BG3ToolConvertJsonsToYaml(BG3ToolPlugin): + icon_file = "ui-next.ico" + sub_name = "Convert JSONS to YAML" + desc = "Convert all jsons in active mods to yaml immediately." + + def display(self): + from ...game_baldursgate3 import BG3Game + + game_plugin = self._organizer.managedGame() + if not isinstance(game_plugin, BG3Game): + return + utils = game_plugin.utils + qInfo("converting all json files to yaml") + active_mods = utils.active_mods() + progress = utils.create_progress_window( + "Converting all json files to yaml", len(active_mods) + 1 + ) + for mod in active_mods: + _convert_jsons_in_dir_to_yaml(Path(mod.absolutePath())) + progress.setValue(progress.value() + 1) + QApplication.processEvents() + if progress.wasCanceled(): + qWarning("conversion canceled by user") + return + _convert_jsons_in_dir_to_yaml(utils.overwrite_path) + progress.setValue(len(active_mods) + 1) + QApplication.processEvents() + progress.close() + + +def _convert_jsons_in_dir_to_yaml(path: Path): + import yaml + + for file in list(path.rglob("*.json")): + converted_path = file.parent / file.name.replace(".json", ".yaml") + try: + if not converted_path.exists() or os.path.getmtime(file) > os.path.getmtime( + converted_path + ): + with open(file, "r") as json_file: + with open(converted_path, "w") as yaml_file: + yaml.dump( + json.load(json_file), yaml_file, indent=2, sort_keys=False + ) + qInfo(f"Converted {file} to YAML") + except OSError as e: + qWarning(f"Error accessing file {converted_path}: {e}") diff --git a/games/baldursgate3/plugins/icons/ui-next.ico b/games/baldursgate3/plugins/icons/ui-next.ico new file mode 100644 index 0000000..1ef6e3b Binary files /dev/null and b/games/baldursgate3/plugins/icons/ui-next.ico differ diff --git a/games/baldursgate3/plugins/icons/ui-refresh.ico b/games/baldursgate3/plugins/icons/ui-refresh.ico new file mode 100644 index 0000000..c018f6c Binary files /dev/null and b/games/baldursgate3/plugins/icons/ui-refresh.ico differ diff --git a/games/baldursgate3/plugins/icons/ui-update.ico b/games/baldursgate3/plugins/icons/ui-update.ico new file mode 100644 index 0000000..5faaf18 Binary files /dev/null and b/games/baldursgate3/plugins/icons/ui-update.ico differ diff --git a/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py b/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py new file mode 100644 index 0000000..4f75371 --- /dev/null +++ b/games/baldursgate3/plugins/reparse_pak_metadata_plugin.py @@ -0,0 +1,16 @@ +from .bg3_tool_plugin import BG3ToolPlugin + + +class BG3ToolReparsePakMetadata(BG3ToolPlugin): + icon_file = "ui-refresh.ico" + sub_name = "Reparse Pak Metadata" + desc = "Force reparsing mod metadata immediately." + + def display(self): + from ...game_baldursgate3 import BG3Game + + game_plugin = self._organizer.managedGame() + if isinstance(game_plugin, BG3Game): + game_plugin.utils.construct_modsettings_xml( + exec_path="bin/bg3", force_reparse_metadata=True + ) diff --git a/games/game_baldursgate3.py b/games/game_baldursgate3.py new file mode 100644 index 0000000..03bd8a0 --- /dev/null +++ b/games/game_baldursgate3.py @@ -0,0 +1,214 @@ +import datetime +import difflib +import os +import shutil +from functools import cached_property +from pathlib import Path +from typing import Any + +from PyQt6.QtCore import ( + qDebug, + qInfo, +) + +import mobase + +from ..basic_features import ( + BasicGameSaveGameInfo, + BasicLocalSavegames, +) +from ..basic_game import BasicGame +from .baldursgate3 import bg3_file_mapper + + +class BG3Game(BasicGame, bg3_file_mapper.BG3FileMapper): + Name = "Baldur's Gate 3 Plugin" + Author = "daescha" + Version = "0.1.0" + GameName = "Baldur's Gate 3" + GameShortName = "baldursgate3" + GameNexusName = "baldursgate3" + GameValidShortNames = ["bg3"] + GameLauncher = "Launcher/LariLauncher.exe" + GameBinary = "bin/bg3.exe" + GameDataPath = "" + GameDocumentsDirectory = ( + "%USERPROFILE%/AppData/Local/Larian Studios/Baldur's Gate 3" + ) + GameSavesDirectory = "%GAME_DOCUMENTS%/PlayerProfiles/Public/Savegames/Story" + GameSaveExtension = "lsv" + GameNexusId = 3474 + GameSteamId = 1086940 + GameGogId = 1456460669 + + def __init__(self): + BasicGame.__init__(self) + from .baldursgate3 import bg3_utils + + self.utils = bg3_utils.BG3Utils(self.name()) + bg3_file_mapper.BG3FileMapper.__init__( + self, self.utils, self.documentsDirectory + ) + + def init(self, organizer: mobase.IOrganizer) -> bool: + super().init(organizer) + self.utils.init(organizer) + from .baldursgate3 import ( + bg3_data_checker, + bg3_data_content, + ) + + self._register_feature(bg3_data_checker.BG3ModDataChecker()) + self._register_feature(bg3_data_content.BG3DataContent()) + self._register_feature(BasicGameSaveGameInfo(lambda s: s.with_suffix(".webp"))) + self._register_feature(BasicLocalSavegames(self.savesDirectory())) + organizer.onAboutToRun(self.utils.construct_modsettings_xml) + organizer.onFinishedRun(self._on_finished_run) + organizer.onUserInterfaceInitialized(self.utils.on_user_interface_initialized) + organizer.modList().onModInstalled(self.utils.on_mod_installed) + organizer.onPluginSettingChanged(self.utils.on_settings_changed) + return True + + def settings(self): + base_settings = super().settings() + custom_settings = [ + mobase.PluginSetting( + "force_load_dlls", + "Force load all dlls detected in active mods. Removes the need for 'Native Mod Loader' and similar mods.", + True, + ), + mobase.PluginSetting( + "log_diff", + "Log a diff of the modsettings.xml file before and after the game runs to check for differences.", + False, + ), + mobase.PluginSetting( + "delete_levelcache_folders_older_than_x_days", + "Maximum number of days a file in overwrite/LevelCache is allowed to exist before being deleted " + "after the executable finishes. Set to negative to disable.", + 3, + ), + mobase.PluginSetting( + "autobuild_paks", + "Autobuild folders likely to be PAK folders with every run of an executable.", + True, + ), + mobase.PluginSetting( + "remove_extracted_metadata", + "Remove extracted meta.lsx files when they are no longer needed.", + True, + ), + mobase.PluginSetting( + "extract_full_package", + "Extract the full pak when parsing metadata, instead of just meta.lsx.", + False, + ), + mobase.PluginSetting( + "convert_yamls_to_json", + "Convert YAMLs to JSONs when executable runs. Allows one to configure ScriptExtender and related mods with YAML files.", + False, + ), + ] + for setting in custom_settings: + setting.description = self.utils.tr(setting.description) + base_settings.append(setting) + return base_settings + + def executables(self) -> list[mobase.ExecutableInfo]: + return [ + mobase.ExecutableInfo( + f"{self.gameName()}: DX11", + self.gameDirectory().absoluteFilePath("bin/bg3_dx11.exe"), + ), + mobase.ExecutableInfo( + f"{self.gameName()}: Vulkan", + self.gameDirectory().absoluteFilePath(self.binaryName()), + ), + mobase.ExecutableInfo( + "Larian Launcher", + self.gameDirectory().absoluteFilePath(self.getLauncherName()), + ), + ] + + def executableForcedLoads(self) -> list[mobase.ExecutableForcedLoadSetting]: + try: + efls = super().executableForcedLoads() + except AttributeError: + efls = [] + if self.utils.force_load_dlls: + qInfo("detecting dlls in enabled mods") + libs: set[str] = set() + tree: mobase.IFileTree | mobase.FileTreeEntry | None = ( + self._organizer.virtualFileTree().find("bin") + ) + if type(tree) is not mobase.IFileTree: + return efls + + def find_dlls( + _: Any, entry: mobase.FileTreeEntry + ) -> mobase.IFileTree.WalkReturn: + relpath = entry.pathFrom(tree) + if ( + relpath + and entry.hasSuffix("dll") + and relpath not in self._base_dlls + ): + libs.add(relpath) + return mobase.IFileTree.WalkReturn.CONTINUE + + tree.walk(find_dlls) + exes = self.executables() + qDebug(f"dlls to force load: {libs}") + efls = efls + [ + mobase.ExecutableForcedLoadSetting( + exe.binary().fileName(), lib + ).withEnabled(True) + for lib in libs + for exe in exes + ] + return efls + + def loadOrderMechanism(self) -> mobase.LoadOrderMechanism: + return mobase.LoadOrderMechanism.PLUGINS_TXT + + @cached_property + def _base_dlls(self) -> set[str]: + base_bin = Path(self.gameDirectory().absoluteFilePath("bin")) + return {str(f.relative_to(base_bin)) for f in base_bin.glob("*.dll")} + + def _on_finished_run(self, exec_path: str, exit_code: int): + if "bin/bg3" not in exec_path: + return + if self.utils.log_diff: + for x in difflib.unified_diff( + open(self.utils.modsettings_backup).readlines(), + open(self.utils.modsettings_path).readlines(), + fromfile=str(self.utils.modsettings_backup), + tofile=str(self.utils.modsettings_path), + lineterm="", + ): + qDebug(x) + for path in self.utils.overwrite_path.rglob("*.log"): + try: + qDebug(f"moving {path} to {self.utils.log_dir}") + shutil.move(path, self.utils.log_dir / path.name) + except PermissionError as e: + qDebug(str(e)) + days = self.utils.get_setting("delete_levelcache_folders_older_than_x_days") + if type(days) is int and days >= 0: + cutoff_time = datetime.datetime.now() - datetime.timedelta(days=days) + qDebug(f"cleaning folders in overwrite/LevelCache older than {cutoff_time}") + for path in self.utils.overwrite_path.glob("LevelCache/*"): + if ( + datetime.datetime.fromtimestamp(os.path.getmtime(path)) + < cutoff_time + ): + shutil.rmtree(path, ignore_errors=True) + qDebug("cleaning empty dirs from overwrite directory") + for folder in sorted( + list(os.walk(self.utils.overwrite_path))[1:], reverse=True + ): + try: + os.rmdir(folder[0]) + except OSError: + pass diff --git a/plugin-requirements.txt b/plugin-requirements.txt index 1f1295e..f945efd 100644 --- a/plugin-requirements.txt +++ b/plugin-requirements.txt @@ -1,3 +1,4 @@ psutil==5.8.0 vdf==3.4 lzokay==1.1.5 +pyyaml==6.0.2 diff --git a/poetry.lock b/poetry.lock index b4ac5a8..a4066dd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -210,7 +210,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -334,4 +334,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "10b882dbbaae72534c631994926861048e8183e18ebbb85c2c46376bdbc70446" +content-hash = "b3a65da37845a56d4b259c4d73913b8957a61382499eeda8bdce2343a48e8c66" diff --git a/pyproject.toml b/pyproject.toml index f5313f5..89778ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ psutil = "^5.8.0" vdf = "3.4" lzokay = "1.1.5" pyqt6 = "6.7.0" +pyyaml = "^6.0.2" [tool.poetry.group.dev.dependencies] mobase-stubs = "^2.5.2"