From 8d3c3b9664a1a9e1b2c4a981449504b5a2db02c4 Mon Sep 17 00:00:00 2001 From: Matt Watson Date: Thu, 24 Apr 2025 15:38:58 -0700 Subject: [PATCH] Fix up package build scripts with better error checking - Better error checking (check we got back the `.whl` files we expect). - Decompose functions a little further. - Fix licensing warning (there's new license names I guess). - Make version file path agree with core Keras. --- RELEASE_PROCESS.md | 4 +- api_gen.py | 4 +- keras_hub/api/__init__.py | 4 +- keras_hub/src/utils/preset_utils.py | 2 +- .../src/{version_utils.py => version.py} | 0 keras_nlp/README.md | 2 +- keras_nlp/pyproject.toml | 8 +- pip_build.py | 151 ++++++++++-------- pyproject.toml | 4 +- 9 files changed, 101 insertions(+), 78 deletions(-) rename keras_hub/src/{version_utils.py => version.py} (100%) diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 9799af8679..70a56ab53c 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -48,7 +48,7 @@ Use the following steps to create an `X.Y.0` release. of the package. Development releases will have version numbers like `X.Y.0.dev0`, and critically will never be installed by default by `pip`. - On the relase branch, check the version in `src/version_utils.py`. If the + On the relase branch, check the version in `src/version.py`. If the current version is not `X.Y.0.dev0`, make a PR following [this template](https://github.com/keras-team/keras-hub/pull/1638) to update the our version number fo look like `X.Y.0.dev0`. This PR should base off our new release branch instead of the master branch. You can use the @@ -164,7 +164,7 @@ to push certain fixes out to our users. of the package. Development releases will have version numbers like `X.Y.Z.dev0`, and critically will never be installed by default by `pip`. - On the relase branch, check the version in `src/version_utils.py`. If the + On the relase branch, check the version in `src/version.py`. If the current version is not `X.Y.Z.dev0`, make a PR following [this template](https://github.com/keras-team/keras-hub/pull/1638) to update the our version number fo look like `X.Y.Z.dev0`. This PR should base off our new release branch instead of the master branch. You can use the diff --git a/api_gen.py b/api_gen.py index 3e104ef8f5..9e2466a122 100644 --- a/api_gen.py +++ b/api_gen.py @@ -36,7 +36,9 @@ def export_version_string(api_init_fname): with open(api_init_fname) as f: contents = f.read() with open(api_init_fname, "w") as f: - contents += "from keras_hub.src.version_utils import __version__ as __version__\n" # noqa: E501 + contents += ( + "from keras_hub.src.version import __version__ as __version__\n" + ) f.write(contents) diff --git a/keras_hub/api/__init__.py b/keras_hub/api/__init__.py index 07bac05324..2aa98bf3f9 100644 --- a/keras_hub/api/__init__.py +++ b/keras_hub/api/__init__.py @@ -11,5 +11,5 @@ from keras_hub import tokenizers as tokenizers from keras_hub import utils as utils from keras_hub.src.utils.preset_utils import upload_preset as upload_preset -from keras_hub.src.version_utils import __version__ as __version__ -from keras_hub.src.version_utils import version as version +from keras_hub.src.version import __version__ as __version__ +from keras_hub.src.version import version as version diff --git a/keras_hub/src/utils/preset_utils.py b/keras_hub/src/utils/preset_utils.py index 7fa4b3bb00..59b68a6dca 100644 --- a/keras_hub/src/utils/preset_utils.py +++ b/keras_hub/src/utils/preset_utils.py @@ -801,7 +801,7 @@ def _save_serialized_object(self, layer, config_file): def _save_metadata(self, layer): from keras_hub.src.models.task import Task - from keras_hub.src.version_utils import __version__ as keras_hub_version + from keras_hub.src.version import __version__ as keras_hub_version # Find all tasks that are compatible with the backbone. # E.g. for `BertBackbone` we would have `TextClassifier` and `MaskedLM`. diff --git a/keras_hub/src/version_utils.py b/keras_hub/src/version.py similarity index 100% rename from keras_hub/src/version_utils.py rename to keras_hub/src/version.py diff --git a/keras_nlp/README.md b/keras_nlp/README.md index 1ef858f088..7f1890d94d 100644 --- a/keras_nlp/README.md +++ b/keras_nlp/README.md @@ -3,5 +3,5 @@ KerasNLP has renamed to KerasHub! Read the announcement [here](https://github.com/keras-team/keras-nlp/issues/1831). -This directory contains a shim package for `keras-nlp` so that the old style +This contains a shim package for `keras-nlp` so that the old style `pip install keras-nlp` and `import keras_nlp` continue to work. diff --git a/keras_nlp/pyproject.toml b/keras_nlp/pyproject.toml index e603d16595..1e737ed5aa 100644 --- a/keras_nlp/pyproject.toml +++ b/keras_nlp/pyproject.toml @@ -4,14 +4,15 @@ build-backend = "setuptools.build_meta" [project] name = "keras-nlp" -version = "0.0.0" # We be replace in pip_build.py +version = "0.0.0" # Will be replaced in pip_build.py +dependencies = ["keras-hub==0.0.0"] # Will be replaced in pip_build.py authors = [ {name = "Keras team", email = "keras-users@googlegroups.com"}, ] description = "Pretrained models for Keras." readme = "README.md" requires-python = ">=3.9" -license = {text = "Apache License 2.0"} +license = "Apache-2.0" classifiers = [ "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3", @@ -26,9 +27,6 @@ classifiers = [ "Topic :: Scientific/Engineering", "Topic :: Software Development", ] -dependencies = [ - "keras-hub==0.0.0", # We be replace in pip_build.py -] [project.urls] Home = "https://keras.io/keras_hub/" diff --git a/pip_build.py b/pip_build.py index 67e43eda36..2803a24a19 100644 --- a/pip_build.py +++ b/pip_build.py @@ -31,6 +31,13 @@ import re import shutil +from keras_hub.src.version import __version__ + + +def ignore_files(_, filenames): + return [f for f in filenames if "_test" in f] + + hub_package = "keras_hub" nlp_package = "keras_nlp" build_directory = "tmp_build_dir" @@ -38,51 +45,65 @@ to_copy = ["pyproject.toml", "README.md"] -def ignore_files(_, filenames): - return [f for f in filenames if "_test" in f] +def update_nightly_version(build_path, version): + """Rewrite library version with the nightly package version.""" + date = datetime.datetime.now() + new_version = re.sub( + r"([0-9]+\.[0-9]+\.[0-9]+).*", # Match version without suffix. + r"\1.dev" + date.strftime("%Y%m%d%H%M"), # Add dev{date} suffix. + version, + ) + + version_file = build_path / hub_package / "src" / "version.py" + version_contents = version_file.read_text() + version_contents = re.sub( + "\n__version__ = .*\n", + f'\n__version__ = "{new_version}"\n', + version_contents, + ) + version_file.write_text(version_contents) + return new_version + + +def update_nightly_name(build_path, pkg_name): + """Rewrite library name with the nightly package name.""" + new_pkg_name = f"{pkg_name}-nightly" + pyproj_file = build_path / "pyproject.toml" + pyproj_contents = pyproj_file.read_text() + pyproj_contents = pyproj_contents.replace( + f'name = "{pkg_name}"', f'name = "{new_pkg_name}"' + ) + pyproj_file.write_text(pyproj_contents) + return new_pkg_name -def update_build_files(build_path, package, version, is_nightly=False): - package_name = package.replace("-", "_") - build_path = pathlib.Path(build_path) +def pin_keras_nlp_version(build_path, pkg_name, version): + """Pin keras-nlp version and dependency to the keras-hub version.""" pyproj_file = build_path / "pyproject.toml" - if is_nightly: - pyproj_contents = pyproj_file.read_text().replace( - f'name = "{package_name}"', f'name = "{package_name}-nightly"' - ) - pyproj_file.write_text(pyproj_contents) - - # Update the version. - if package == hub_package: - # KerasHub pyproject reads the version dynamically from source. - version_file = build_path / package / "src" / "version_utils.py" - version_contents = version_file.read_text() - version_contents = re.sub( - "\n__version__ = .*\n", - f'\n__version__ = "{version}"\n', - version_contents, - ) - version_file.write_text(version_contents) - elif package == nlp_package: - # For the KerasNLP shim we need to replace the version in the pyproject - # file, so we can pin the version of the keras-hub in dependencies. - pyproj_str = pyproj_file.read_text().replace("0.0.0", version) - pyproj_file.write_text(pyproj_str) - - -def copy_source_to_build_directory(root_path, package): + pyproj_contents = pyproj_file.read_text() + pyproj_contents = re.sub( + "version = .*\n", + f'version = "{version}"\n', + pyproj_contents, + ) + + pyproj_contents = re.sub( + "dependencies = .*\n", + f'dependencies = ["{pkg_name}=={version}"]\n', + pyproj_contents, + ) + pyproj_file.write_text(pyproj_contents) + + +def copy_source_to_build_directory(src, dst, package): # Copy sources (`keras_hub/` directory and setup files) to build # directory - shutil.copytree( - root_path / package, - root_path / build_directory / package, - ignore=ignore_files, - ) + shutil.copytree(src / package, dst / package, ignore=ignore_files) for fname in to_copy: - shutil.copy(root_path / fname, root_path / build_directory / fname) + shutil.copy(src / fname, dst / fname) -def build_wheel(build_path, dist_path, version): +def build_wheel(build_path, dist_path, name, version): # Build the package os.chdir(build_path) os.system("python3 -m build") @@ -93,50 +114,52 @@ def build_wheel(build_path, dist_path, version): for fpath in (build_path / dist_directory).glob("*.*"): shutil.copy(fpath, dist_path) - # Find the .whl file path - for fname in os.listdir(dist_path): - if version in fname and fname.endswith(".whl"): - whl_path = dist_path / fname - print(f"Build successful. Wheel file available at {whl_path}") - return whl_path - print("Build failed.") - return None + # Check for the expected .whl file path + name = name.replace("-", "_") + whl_path = dist_path / f"{name}-{version}-py3-none-any.whl" + if not os.path.exists(whl_path): + raise ValueError(f"Could not find whl {whl_path}") + print(f"Build successful. Wheel file available at {whl_path}") + return whl_path def build(root_path, is_nightly=False, keras_nlp=True): if os.path.exists(build_directory): raise ValueError(f"Directory already exists: {build_directory}") - from keras_hub.src.version_utils import __version__ # noqa: E402 - - if is_nightly: - date = datetime.datetime.now() - version = re.sub( - r"([0-9]+\.[0-9]+\.[0-9]+).*", # Match version without suffix. - r"\1.dev" + date.strftime("%Y%m%d%H%M"), # Add dev{date} suffix. - __version__, - ) - else: - version = __version__ - try: whls = [] build_path = root_path / build_directory dist_path = root_path / dist_directory os.mkdir(build_path) + copy_source_to_build_directory(root_path, build_path, hub_package) - copy_source_to_build_directory(root_path, hub_package) - update_build_files(build_path, hub_package, version, is_nightly) - whl = build_wheel(build_path, dist_path, version) + version = __version__ + pkg_name = hub_package.replace("_", "-") + if is_nightly: + version = update_nightly_version(build_path, version) + pkg_name = update_nightly_name(build_path, pkg_name) + assert "dev" in version, "Version should contain dev" + assert "nightly" in pkg_name, "Name should contain nightly" + + whl = build_wheel(build_path, dist_path, pkg_name, version) whls.append(whl) if keras_nlp: build_path = root_path / build_directory / nlp_package dist_path = root_path / nlp_package / dist_directory - - copy_source_to_build_directory(root_path, nlp_package) - update_build_files(build_path, nlp_package, version, is_nightly) - whl = build_wheel(build_path, dist_path, version) + copy_source_to_build_directory( + root_path / nlp_package, build_path, nlp_package + ) + + pin_keras_nlp_version(build_path, pkg_name, version) + nlp_pkg_name = nlp_package.replace("_", "-") + if is_nightly: + nlp_pkg_name = update_nightly_name(build_path, nlp_pkg_name) + assert "dev" in version, "Version should contain dev" + assert "nightly" in nlp_pkg_name, "Name should contain nightly" + + whl = build_wheel(build_path, dist_path, nlp_pkg_name, version) whls.append(whl) return whls diff --git a/pyproject.toml b/pyproject.toml index 79cc80d5d3..1ed8b459b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ description = "Pretrained models for Keras." readme = "README.md" requires-python = ">=3.9" -license = {text = "Apache License 2.0"} +license = "Apache-2.0" dynamic = ["version"] classifiers = [ "Development Status :: 3 - Alpha", @@ -42,7 +42,7 @@ Home = "https://keras.io/keras_hub/" Repository = "https://github.com/keras-team/keras/keras_hub" [tool.setuptools.dynamic] -version = {attr = "keras_hub.src.version_utils.__version__"} +version = {attr = "keras_hub.src.version.__version__"} [tool.setuptools.package-dir] "" = "."