Merge branch 'main' into fix-install-stanza

Author: kellyma2

.bazelci/tests.yml

@@ -33,6 +33,10 @@ default_tests_with_rpm: &default_tests_with_rpm
     - "//toolchains/..."
 
 win_tests: &win_tests
+  test_flags:
+    - "--test_tag_filters=-nowindows"
+  build_flags:
+    - "--build_tag_filters=-nowindows"
   test_targets:
     - "//pkg/..."
     - "//tests/..."

distro/BUILD

@@ -84,6 +84,7 @@ py_test(
         "packaging_test.py",
         ":release_version.py",
     ],
+    imports = [".."],
     data = [
         "testdata/BUILD.tpl",
         ":distro",

docs/latest.md

@@ -259,8 +259,8 @@ Rules for making .tar files.
 ## pkg_tar
 
 <pre>
-pkg_tar(<a href="#pkg_tar-name">name</a>, <a href="#pkg_tar-build_tar">build_tar</a>, <a href="#pkg_tar-compressor">compressor</a>, <a href="#pkg_tar-compressor_args">compressor_args</a>, <a href="#pkg_tar-deps">deps</a>, <a href="#pkg_tar-empty_dirs">empty_dirs</a>, <a href="#pkg_tar-empty_files">empty_files</a>, <a href="#pkg_tar-extension">extension</a>,
-             <a href="#pkg_tar-files">files</a>, <a href="#pkg_tar-include_runfiles">include_runfiles</a>, <a href="#pkg_tar-mode">mode</a>, <a href="#pkg_tar-modes">modes</a>, <a href="#pkg_tar-mtime">mtime</a>, <a href="#pkg_tar-out">out</a>, <a href="#pkg_tar-owner">owner</a>, <a href="#pkg_tar-ownername">ownername</a>, <a href="#pkg_tar-ownernames">ownernames</a>, <a href="#pkg_tar-owners">owners</a>,
+pkg_tar(<a href="#pkg_tar-name">name</a>, <a href="#pkg_tar-compressor">compressor</a>, <a href="#pkg_tar-compressor_args">compressor_args</a>, <a href="#pkg_tar-deps">deps</a>, <a href="#pkg_tar-empty_dirs">empty_dirs</a>, <a href="#pkg_tar-empty_files">empty_files</a>, <a href="#pkg_tar-extension">extension</a>, <a href="#pkg_tar-files">files</a>,
+             <a href="#pkg_tar-include_runfiles">include_runfiles</a>, <a href="#pkg_tar-mode">mode</a>, <a href="#pkg_tar-modes">modes</a>, <a href="#pkg_tar-mtime">mtime</a>, <a href="#pkg_tar-out">out</a>, <a href="#pkg_tar-owner">owner</a>, <a href="#pkg_tar-ownername">ownername</a>, <a href="#pkg_tar-ownernames">ownernames</a>, <a href="#pkg_tar-owners">owners</a>,
              <a href="#pkg_tar-package_dir">package_dir</a>, <a href="#pkg_tar-package_dir_file">package_dir_file</a>, <a href="#pkg_tar-package_file_name">package_file_name</a>, <a href="#pkg_tar-package_variables">package_variables</a>, <a href="#pkg_tar-portable_mtime">portable_mtime</a>,
              <a href="#pkg_tar-private_stamp_detect">private_stamp_detect</a>, <a href="#pkg_tar-remap_paths">remap_paths</a>, <a href="#pkg_tar-srcs">srcs</a>, <a href="#pkg_tar-stamp">stamp</a>, <a href="#pkg_tar-strip_prefix">strip_prefix</a>, <a href="#pkg_tar-symlinks">symlinks</a>)
 </pre>
@@ -273,31 +273,30 @@ pkg_tar(<a href="#pkg_tar-name">name</a>, <a href="#pkg_tar-build_tar">build_tar
 | Name  | Description | Type | Mandatory | Default |
 | :------------- | :------------- | :------------- | :------------- | :------------- |
 | <a id="pkg_tar-name"></a>name |  A unique name for this target.   | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required |  |
-| <a id="pkg_tar-build_tar"></a>build_tar |  -   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | //pkg/private/tar:build_tar |
-| <a id="pkg_tar-compressor"></a>compressor |  -   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
-| <a id="pkg_tar-compressor_args"></a>compressor_args |  -   | String | optional | "" |
-| <a id="pkg_tar-deps"></a>deps |  -   | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
+| <a id="pkg_tar-compressor"></a>compressor |  External tool which can compress the archive.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="pkg_tar-compressor_args"></a>compressor_args |  Arg list for <code>compressor</code>.   | String | optional | "" |
+| <a id="pkg_tar-deps"></a>deps |  tar files which will be unpacked and repacked into the archive.   | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
 | <a id="pkg_tar-empty_dirs"></a>empty_dirs |  -   | List of strings | optional | [] |
 | <a id="pkg_tar-empty_files"></a>empty_files |  -   | List of strings | optional | [] |
 | <a id="pkg_tar-extension"></a>extension |  -   | String | optional | "tar" |
-| <a id="pkg_tar-files"></a>files |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: Label -> String</a> | optional | {} |
+| <a id="pkg_tar-files"></a>files |  Obsolete. Do not use.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: Label -> String</a> | optional | {} |
 | <a id="pkg_tar-include_runfiles"></a>include_runfiles |  -   | Boolean | optional | False |
 | <a id="pkg_tar-mode"></a>mode |  -   | String | optional | "0555" |
 | <a id="pkg_tar-modes"></a>modes |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
 | <a id="pkg_tar-mtime"></a>mtime |  -   | Integer | optional | -1 |
 | <a id="pkg_tar-out"></a>out |  -   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | required |  |
-| <a id="pkg_tar-owner"></a>owner |  -   | String | optional | "0.0" |
+| <a id="pkg_tar-owner"></a>owner |  Default numeric owner.group to apply to files when not set via pkg_attribures.   | String | optional | "0.0" |
 | <a id="pkg_tar-ownername"></a>ownername |  -   | String | optional | "." |
 | <a id="pkg_tar-ownernames"></a>ownernames |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
 | <a id="pkg_tar-owners"></a>owners |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
-| <a id="pkg_tar-package_dir"></a>package_dir |  Prefix to be prepend to all paths written. The name may contain variables, same as [package_file_name](#package_file_name)   | String | optional | "" |
+| <a id="pkg_tar-package_dir"></a>package_dir |  Prefix to be prepend to all paths written.<br><br>            This is applied as a final step, while writing to the archive.             Any other attributes (e.g. symlinks) which specify a path, must do so relative to package_dir.             The value may contain variables. See [package_file_name](#package_file_name) for examples.   | String | optional | "" |
 | <a id="pkg_tar-package_dir_file"></a>package_dir_file |  -   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
 | <a id="pkg_tar-package_file_name"></a>package_file_name |  See [Common Attributes](#package_file_name)   | String | optional | "" |
 | <a id="pkg_tar-package_variables"></a>package_variables |  See [Common Attributes](#package_variables)   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
 | <a id="pkg_tar-portable_mtime"></a>portable_mtime |  -   | Boolean | optional | True |
 | <a id="pkg_tar-private_stamp_detect"></a>private_stamp_detect |  -   | Boolean | optional | False |
 | <a id="pkg_tar-remap_paths"></a>remap_paths |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
-| <a id="pkg_tar-srcs"></a>srcs |  -   | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
+| <a id="pkg_tar-srcs"></a>srcs |  Inputs which will become part of the tar archive.   | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
 | <a id="pkg_tar-stamp"></a>stamp |  Enable file time stamping.  Possible values: <li>stamp = 1: Use the time of the build as the modification time of each file in the archive. <li>stamp = 0: Use an "epoch" time for the modification time of each file. This gives good build result caching. <li>stamp = -1: Control the chosen modification time using the --[no]stamp flag. <div class="since"><i>Since 0.5.0</i></div>   | Integer | optional | 0 |
 | <a id="pkg_tar-strip_prefix"></a>strip_prefix |  (note: Use strip_prefix = "." to strip path to the package but preserve relative paths of sub directories beneath the package.)   | String | optional | "" |
 | <a id="pkg_tar-symlinks"></a>symlinks |  -   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
@@ -583,6 +582,9 @@ pkg_mklink(<a href="#pkg_mklink-name">name</a>, <a href="#pkg_mklink-link_name">
 
 Create a symlink.
 
+Wraps [pkg_mklink_impl](#pkg_mklink_impl)
+
+
 **PARAMETERS**
 
 

pkg/BUILD

@@ -68,6 +68,7 @@ starlark_library(
 py_binary(
     name = "make_rpm",
     srcs = ["make_rpm.py"],
+    imports = [".."],
     python_version = "PY3",
     srcs_version = "PY3",
     target_compatible_with = select({
@@ -84,6 +85,7 @@ py_binary(
 py_library(
     name = "make_rpm_lib",
     srcs = ["make_rpm.py"],
+    imports = [".."],
     srcs_version = "PY3",
     visibility = [
         "//experimental:__subpackages__",

pkg/make_rpm.py

@@ -70,7 +70,8 @@ def Tempdir():
   dirpath = tempfile.mkdtemp()
 
   def Cleanup():
-    shutil.rmtree(dirpath)
+#    shutil.rmtree(dirpath)
+     pass
 
   with Cd(dirpath, Cleanup):
     yield dirpath
@@ -81,10 +82,9 @@ def Cleanup():
 
 def FindOutputFile(log):
   """Find the written file from the log information."""
-
-  m = WROTE_FILE_RE.search(log)
+  m = WROTE_FILE_RE.findall(log)
   if m:
-    return m.group('rpm_path')
+    return m
   return None
 
 def SlurpFile(input_path):
@@ -187,7 +187,7 @@ def __init__(self, name, version, release, arch, rpmbuild_path,
     self.arch = arch
     self.files = []
     self.rpmbuild_path = FindRpmbuild(rpmbuild_path)
-    self.rpm_path = None
+    self.rpm_paths = None
     self.source_date_epoch = helpers.GetFlagValue(source_date_epoch)
     self.debug = debug
 
@@ -204,6 +204,7 @@ def __init__(self, name, version, release, arch, rpmbuild_path,
     self.post_scriptlet = None
     self.preun_scriptlet = None
     self.postun_scriptlet = None
+    self.subpackages = None
 
   def AddFiles(self, paths, root=''):
     """Add a set of files to the current RPM.
@@ -227,6 +228,7 @@ def SetupWorkdir(self,
                    preamble_file=None,
                    description_file=None,
                    install_script_file=None,
+                   subpackages_file=None,
                    pre_scriptlet_path=None,
                    post_scriptlet_path=None,
                    preun_scriptlet_path=None,
@@ -240,6 +242,10 @@ def SetupWorkdir(self,
       if not os.path.exists(name):
         os.makedirs(name, 0o777)
 
+    # create the bld directory structure
+    package_bld_dir = "bld-%s/%s" % ( self.arch, self.name )
+    os.makedirs(package_bld_dir, 0o777)
+
     # Copy the to-be-packaged files into the BUILD directory
     for f in self.files:
       dst_dir = os.path.join(RpmBuilder.BUILD_DIR, os.path.dirname(f))
@@ -264,14 +270,17 @@ def SetupWorkdir(self,
       SlurpFile(os.path.join(original_dir, preun_scriptlet_path)) if preun_scriptlet_path is not None else ''
     self.postun_scriptlet = \
       SlurpFile(os.path.join(original_dir, postun_scriptlet_path)) if postun_scriptlet_path is not None else ''
+    self.subpackages = \
+      SlurpFile(os.path.join(original_dir, subpackages_file)) if subpackages_file is not None else ''
 
     # Then prepare for textual substitution.  This is typically only the case for the
     # experimental `pkg_rpm`.
     tpl_replacements = {
-      'PRE_SCRIPTLET': "%pre\n" + self.pre_scriptlet,
-      'POST_SCRIPTLET': "%post\n" + self.post_scriptlet,
-      'PREUN_SCRIPTLET': "%preun\n" + self.preun_scriptlet,
-      'POSTUN_SCRIPTLET': "%postun\n" + self.postun_scriptlet,
+      'PRE_SCRIPTLET': ("%pre\n" + self.pre_scriptlet) if self.pre_scriptlet else "",
+      'POST_SCRIPTLET': ("%post\n" + self.post_scriptlet) if self.post_scriptlet else "",
+      'PREUN_SCRIPTLET': ("%preun\n" + self.preun_scriptlet) if self.preun_scriptlet else "",
+      'POSTUN_SCRIPTLET': ("%postun\n" + self.postun_scriptlet) if self.postun_scriptlet else "",
+      'SUBPACKAGES' : (self.subpackages if self.subpackages else ""),
       'CHANGELOG': ""
     }
 
@@ -334,8 +343,8 @@ def SetupWorkdir(self,
 
     # Used in %files -f
     if file_list_path:
-      shutil.copy(os.path.join(original_dir, file_list_path), RpmBuilder.BUILD_DIR)
-      self.file_list_path = os.path.join(RpmBuilder.BUILD_DIR, os.path.basename(file_list_path))
+      shutil.copy(os.path.join(original_dir, file_list_path), package_bld_dir)
+      self.file_list_path = os.path.join(package_bld_dir, os.path.basename(file_list_path))
 
   def CallRpmBuild(self, dirname, rpmbuild_args):
     """Call rpmbuild with the correct arguments."""
@@ -391,6 +400,7 @@ def CallRpmBuild(self, dirname, rpmbuild_args):
       print('With environment:')
       pprint.pprint(env)
 
+    args += [ '--verbose' ]
     p = subprocess.Popen(
         args,
         stdout=subprocess.PIPE,
@@ -400,9 +410,9 @@ def CallRpmBuild(self, dirname, rpmbuild_args):
 
     if p.returncode == 0:
       # Find the created file.
-      self.rpm_path = FindOutputFile(output)
+      self.rpm_paths = FindOutputFile(output)
 
-    if p.returncode != 0 or not self.rpm_path:
+    if p.returncode != 0 or not self.rpm_paths:
       print('Error calling rpmbuild:')
       print(output)
     elif self.debug:
@@ -411,20 +421,33 @@ def CallRpmBuild(self, dirname, rpmbuild_args):
     # Return the status.
     return p.returncode
 
-  def SaveResult(self, out_file):
+  def SaveResult(self, out_file, subpackage_out_files):
     """Save the result RPM out of the temporary working directory."""
 
-    if self.rpm_path:
-      shutil.copy(self.rpm_path, out_file)
+    if self.rpm_paths:
+      rpm_path = self.rpm_paths[0]
+      shutil.copy(rpm_path, out_file)
       if self.debug:
         print('Saved RPM file to %s' % out_file)
+
+      subrpm_paths = self.rpm_paths[1:]
+      for s_name, s_out_file in subpackage_out_files:
+         s_prefix = self.name + '-' + s_name
+
+         for p in subrpm_paths:
+            if os.path.basename(p).startswith(s_prefix):
+               shutil.copy(p, s_out_file)
+               if self.debug:
+                  print('Saved %s sub RPM file to %s' % (s_name, s_out_file))
+               break
     else:
       print('No RPM file created.')
 
-  def Build(self, spec_file, out_file,
+  def Build(self, spec_file, out_file, subpackage_out_files,
             preamble_file=None,
             description_file=None,
             install_script_file=None,
+            subpackages_file=None,
             pre_scriptlet_path=None,
             post_scriptlet_path=None,
             preun_scriptlet_path=None,
@@ -440,20 +463,24 @@ def Build(self, spec_file, out_file,
     original_dir = os.getcwd()
     spec_file = os.path.join(original_dir, spec_file)
     out_file = os.path.join(original_dir, out_file)
+    subpackage_out_files = (s.split(':') for s in subpackage_out_files)
+    subpackage_out_files = [
+       (s[0], os.path.join(original_dir, s[1])) for s in subpackage_out_files]
     with Tempdir() as dirname:
       self.SetupWorkdir(spec_file,
                         original_dir,
                         preamble_file=preamble_file,
                         description_file=description_file,
                         install_script_file=install_script_file,
+                        subpackages_file=subpackages_file,
                         file_list_path=file_list_path,
                         pre_scriptlet_path=pre_scriptlet_path,
                         post_scriptlet_path=post_scriptlet_path,
                         preun_scriptlet_path=preun_scriptlet_path,
                         postun_scriptlet_path=postun_scriptlet_path,
                         changelog_file=changelog_file)
       status = self.CallRpmBuild(dirname, rpmbuild_args or [])
-      self.SaveResult(out_file)
+      self.SaveResult(out_file, subpackage_out_files)
 
     return status
 
@@ -476,6 +503,9 @@ def main(argv):
                       help='The file containing the RPM specification.')
   parser.add_argument('--out_file', required=True,
                       help='The destination to save the resulting RPM file to.')
+  parser.add_argument('--subpackage_out_file', action='append',
+                      help='List of destinations to save resulting ' +
+                      'subpackage RPMs to in the form of name:destination')
   parser.add_argument('--rpmbuild', help='Path to rpmbuild executable.')
   parser.add_argument('--source_date_epoch',
                       help='Value for the SOURCE_DATE_EPOCH rpmbuild '
@@ -492,6 +522,8 @@ def main(argv):
                       help='File containing the RPM Preamble')
   parser.add_argument('--description',
                       help='File containing the RPM %description text')
+  parser.add_argument('--subpackages',
+                      help='File containing the RPM subpackage details')
   parser.add_argument('--pre_scriptlet',
                       help='File containing the RPM %pre scriptlet, if to be substituted')
   parser.add_argument('--post_scriptlet',
@@ -510,16 +542,23 @@ def main(argv):
   options = parser.parse_args(argv or ())
 
   try:
+    arch = options.arch
+    if not arch:
+      import platform
+      arch = platform.machine()
+
     builder = RpmBuilder(options.name,
                          options.version, options.release,
                          options.arch, options.rpmbuild,
                          source_date_epoch=options.source_date_epoch,
                          debug=options.debug)
     builder.AddFiles(options.files)
     return builder.Build(options.spec_file, options.out_file,
+                         options.subpackage_out_file,
                          preamble_file=options.preamble,
                          description_file=options.description,
                          install_script_file=options.install_script,
+                         subpackages_file=options.subpackages,
                          file_list_path=options.file_list,
                          pre_scriptlet_path=options.pre_scriptlet,
                          post_scriptlet_path=options.post_scriptlet,

pkg/private/BUILD

@@ -65,6 +65,7 @@ py_library(
     srcs = [
         "build_info.py",
     ],
+    imports = ["../.."],
     srcs_version = "PY3",
     visibility = [
         "//:__subpackages__",
@@ -78,6 +79,7 @@ py_library(
         "__init__.py",
         "archive.py",
     ],
+    imports = ["../.."],
     srcs_version = "PY3",
     visibility = [
         "//:__subpackages__",
@@ -91,6 +93,7 @@ py_library(
         "__init__.py",
         "helpers.py",
     ],
+    imports = ["../.."],
     srcs_version = "PY3",
     visibility = [
         "//:__subpackages__",
@@ -101,6 +104,7 @@ py_library(
 py_library(
     name = "manifest",
     srcs = ["manifest.py"],
+    imports = ["../.."],
     srcs_version = "PY3",
     visibility = ["//visibility:public"],
 )

pkg/private/deb/BUILD

@@ -48,6 +48,7 @@ exports_files(
 py_binary(
     name = "make_deb",
     srcs = ["make_deb.py"],
+    imports = ["../../.."],
     python_version = "PY3",
     visibility = ["//visibility:public"],
     deps = [
@@ -58,6 +59,7 @@ py_binary(
 py_library(
     name = "make_deb_lib",
     srcs = ["make_deb.py"],
+    imports = ["../../.."],
     srcs_version = "PY3",
     visibility = ["//tests/deb:__pkg__"],
     deps = [

pkg/private/deb/deb.bzl

@@ -16,7 +16,7 @@
 load("//pkg:providers.bzl", "PackageVariablesInfo")
 load("//pkg/private:util.bzl", "setup_output_files")
 
-_tar_filetype = [".tar", ".tar.gz", ".tgz", ".tar.bz2", "tar.xz"]
+_tar_filetype = [".tar", ".tar.gz", ".tgz", ".tar.bz2", "tar.xz", "tar.zst"]
 
 def _pkg_deb_impl(ctx):
     """The implementation for the pkg_deb rule."""

pkg/private/deb/make_deb.py

@@ -237,7 +237,7 @@ def CreateDeb(output,
       ext = 'tar.bz2'
     else:
       ext = '.'.join(ext)
-      if ext not in ['tar.bz2', 'tar.gz', 'tar.xz', 'tar.lzma']:
+      if ext not in ['tar.bz2', 'tar.gz', 'tar.xz', 'tar.lzma', 'tar.zst']:
         ext = 'tar'
     data_size = os.stat(data).st_size
     with open(data, 'rb') as datafile: