Skip to content

introduce XORG_*_URLS and GNU_FTP_URLS template constants #4935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

introduce XORG_*_URLS and GNU_FTP_URLS template constants #4935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9ae700d
update XORG_*_SOURCE constants to use new URL https://www.x.org/archi…
jfgrimm Jun 24, 2025
e530513
introduce XORG_*_URLS and GNU_*_URLS template constants
jfgrimm Jun 25, 2025
ab703dd
fix conflicts
jfgrimm Jun 25, 2025
918d92c
account for lists in reverse TEMPLATE_CONSTANTS mapping
jfgrimm Jun 25, 2025
503ab3e
update unit test
jfgrimm Jun 25, 2025
0a0fa71
update use of GNU_SOURCE
jfgrimm Jun 25, 2025
3f39f32
update test easyconfigs
jfgrimm Jun 25, 2025
fcee672
hold off on deprecating old XORG_*_SOURCE and GNU_*SOURCE template co…
jfgrimm Jul 3, 2025
09dd553
Revert "hold off on deprecating old XORG_*_SOURCE and GNU_*SOURCE tem…
jfgrimm Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion easybuild/framework/easyconfig/easyconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1339,7 +1339,14 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
default_values.update({key: value[0] for key, value in self.extra_options.items()})

self.generate_template_values()
templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()}

# take into account that some values may be lists
templ_const = {}
for name, (value, _) in TEMPLATE_CONSTANTS.items():
if isinstance(value, list):
templ_const.update({quote_py_str(v): name for v in value})
else:
templ_const.update({quote_py_str(value): name})

# create reverse map of templates, to inject template values where possible
# longer template values are considered first, shorter template keys get preference over longer ones
Expand Down
67 changes: 35 additions & 32 deletions easybuild/framework/easyconfig/templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,15 @@
TEMPLATE_CONSTANTS = {
# source url constants
'APACHE_SOURCE': ('https://archive.apache.org/dist/%(namelower)s',
'apache.org source url'),
'apache.org source URL'),
'BITBUCKET_SOURCE': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get',
'bitbucket.org source url '
'bitbucket.org source URL '
'(namelower is used if bitbucket_account easyconfig parameter is not specified)'),
'BITBUCKET_DOWNLOADS': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads',
'bitbucket.org downloads url '
'bitbucket.org downloads URL '
'(namelower is used if bitbucket_account easyconfig parameter is not specified)'),
'CRAN_SOURCE': ('https://cran.r-project.org/src/contrib',
'CRAN (contrib) source url'),
'CRAN (contrib) source URL'),
'FTPGNOME_SOURCE': ('https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s',
'http download for gnome ftp server'),
'GITHUB_SOURCE': ('https://github.com/%(github_account)s/%(name)s/archive',
Expand All @@ -139,33 +139,36 @@
'GitHub release URL with lowercase name (if github_account easyconfig '
'parameter is not specified, namelower is used in its place)'),
'GNU_SAVANNAH_SOURCE': ('https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
'download.savannah.gnu.org source url'),
'GNU_SOURCE': ('https://ftpmirror.gnu.org/gnu/%(namelower)s',
'gnu.org source url (ftp mirror)'),
'GNU_FTP_SOURCE': ('https://ftp.gnu.org/gnu/%(namelower)s',
'gnu.org source url (main ftp)'),
'download.savannah.gnu.org source URL'),
'GNU_FTP_URLS': (['https://ftpmirror.gnu.org/gnu/%(namelower)s', 'https://ftp.gnu.org/gnu/%(namelower)s'],
'gnu.org source URLs (ftp mirror and main)'),
'GOOGLECODE_SOURCE': ('http://%(namelower)s.googlecode.com/files',
'googlecode.com source url'),
'googlecode.com source URL'),
'LAUNCHPAD_SOURCE': ('https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/',
'launchpad.net source url'),
'launchpad.net source URL'),
'PYPI_SOURCE': ('https://pypi.python.org/packages/source/%(nameletter)s/%(name)s',
'pypi source url'), # e.g., Cython, Sphinx
'pypi source URL'), # e.g., Cython, Sphinx
'PYPI_LOWER_SOURCE': ('https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s',
'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ
'pypi source URL (lowercase name)'), # e.g., Greenlet, PyZMQ
'R_SOURCE': ('https://cran.r-project.org/src/base/R-%(version_major)s',
'cran.r-project.org (base) source url'),
'cran.r-project.org (base) source URL'),
'SOURCEFORGE_SOURCE': ('https://download.sourceforge.net/%(namelower)s',
'sourceforge.net source url'),
'XORG_DATA_SOURCE': ('https://xorg.freedesktop.org/archive/individual/data/',
'xorg data source url'),
'XORG_LIB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/lib/',
'xorg lib source url'),
'XORG_PROTO_SOURCE': ('https://xorg.freedesktop.org/archive/individual/proto/',
'xorg proto source url'),
'XORG_UTIL_SOURCE': ('https://xorg.freedesktop.org/archive/individual/util/',
'xorg util source url'),
'XORG_XCB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/xcb/',
'xorg xcb source url'),
'sourceforge.net source URL'),
'XORG_DATA_URLS': (['https://www.x.org/archive/individual/data/',
'https://xorg.freedesktop.org/archive/individual/data/'],
'xorg data source URLs'),
'XORG_LIB_URLS': (['https://www.x.org/archive/individual/lib/',
'https://xorg.freedesktop.org/archive/individual/lib/'],
'xorg lib source URLs'),
'XORG_PROTO_URLS': (['https://www.x.org/archive/individual/proto/',
'https://xorg.freedesktop.org/archive/individual/proto/'],
'xorg proto source URLs'),
'XORG_UTIL_URLS': (['https://www.x.org/archive/individual/util/',
'https://xorg.freedesktop.org/archive/individual/util/'],
'xorg util source URLs'),
'XORG_XCB_URLS': (['https://www.x.org/archive/individual/xcb/',
'https://xorg.freedesktop.org/archive/individual/xcb/'],
'xorg xcb source URLs'),

# TODO, not urgent, yet nice to have:
# CPAN_SOURCE GNOME KDE_I18N XCONTRIB DEBIAN KDE GENTOO TEX_CTAN MOZILLA_ALL
Expand Down Expand Up @@ -224,19 +227,12 @@
'GITHUB_RELEASE_URL': 'GITHUB_RELEASE',
'GITHUB_RELEASE_URL_LOWER': 'GITHUB_LOWER_RELEASE',
'GNU_SAVANNAH_URL': 'GNU_SAVANNAH_SOURCE',
'GNU_FTP_URL': 'GNU_FTP_SOURCE',
'GNU_URL': 'GNU_SOURCE',
'GOOGLECODE_URL': 'GOOGLECODE_SOURCE',
'LAUNCHPAD_URL': 'LAUNCHPAD_SOURCE',
'PYPI_URL': 'PYPI_SOURCE',
'PYPI_URL_LOWER': 'PYPI_LOWER_SOURCE',
'R_URL': 'R_SOURCE',
'SOURCEFORGE_URL': 'SOURCEFORGE_SOURCE',
'XORG_DATA_URL': 'XORG_DATA_SOURCE',
'XORG_LIB_URL': 'XORG_LIB_SOURCE',
'XORG_PROTO_URL': 'XORG_PROTO_SOURCE',
'XORG_UTIL_URL': 'XORG_UTIL_SOURCE',
'XORG_XCB_URL': 'XORG_XCB_SOURCE',
'SOURCE_LOWER_TAR_GZ': 'SOURCELOWER_TAR_GZ',
'SOURCE_LOWER_TAR_XZ': 'SOURCELOWER_TAR_XZ',
'SOURCE_LOWER_TAR_BZ2': 'SOURCELOWER_TAR_BZ2',
Expand All @@ -257,6 +253,13 @@
# deprecated template constants, and their replacements
DEPRECATED_EASYCONFIG_TEMPLATE_CONSTANTS = {
# <old_template_constant>: (<new_template_constant>, <deprecation_version>),
'GNU_FTP_SOURCE': ('GNU_FTP_URLS', '6.0'),
'GNU_SOURCE': ('GNU_FTP_URLS', '6.0'),
'XORG_DATA_SOURCE': ('XORG_DATA_URLS', '6.0'),
'XORG_LIB_SOURCE': ('XORG_LIB_URLS', '6.0'),
'XORG_PROTO_SOURCE': ('XORG_PROTO_URLS', '6.0'),
'XORG_UTIL_SOURCE': ('XORG_PROTO_URLS', '6.0'),
'XORG_XCB_SOURCE': ('XORG_XCB_SOURCE', '6.0'),
}

EXTENSIONS = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z']
Expand Down
4 changes: 2 additions & 2 deletions test/framework/easyconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1305,10 +1305,10 @@ def test_templating_constants(self):

def test_template_constant_import(self):
"""Test importing template constants works"""
from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT
from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_FTP_URLS, SHLIB_EXT
from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0])
self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0])
self.assertEqual(GNU_FTP_URLS, TEMPLATE_CONSTANTS['GNU_FTP_URLS'][0])
self.assertEqual(SHLIB_EXT, get_shared_lib_ext())

def test_ec_method_resolve_template(self):
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigparser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ def test_easyconfig_constants(self):
val = constants[constant_name]
fail_msg = "The constant %s should have an acceptable type, found %s (%s)" % (constant_name,
type(val), str(val))
self.assertIsInstance(val, (str, dict, tuple), fail_msg)
self.assertIsInstance(val, (str, dict, list, tuple), fail_msg)

# check a couple of randomly picked constant values
self.assertEqual(constants['SOURCE_TAR_GZ'], '%(name)s-%(version)s.tar.gz')
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ description = "binutils: GNU binary utilities"
toolchain = {'name': 'GCCcore', 'version': '4.9.3'}

sources = [SOURCE_TAR_GZ]
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS

# Testing purposes only so remove deps
#builddependencies = [
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ description = "binutils: GNU binary utilities"
toolchain = {'name': 'GCCcore', 'version': '4.9.3'}

sources = [SOURCE_TAR_GZ]
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS

# Testing purposes only so remove deps
#builddependencies = [
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ toolchain = {'name': 'GCC', 'version': '4.6.3'}
sources = ['%(name)s-%(version)s.tar.gz']

# download location for source files
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS

hiddendependencies = [('toy', '0.0', '-deps', SYSTEM)]
dependencies = hiddendependencies # hidden deps must be included in list of deps
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}
sources = ['%(name)s-%(version)s.tar.gz']

# download location for source files
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]

# download location for source files
source_urls = ['https://ftpmirror.gnu.org/gzip']
source_urls = ['https://ftpmirror.gnu.org/gzip',
'https://ftp.gnu.org/gnu/gzip']

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-foss-2018a.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ description = "gzip (GNU zip) is a popular data compression program as a replace
toolchain = {'name': 'foss', 'version': '2018a'}

# eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz
source_urls = ['http://ftpmirror.gnu.org/gzip']
source_urls = ['http://ftpmirror.gnu.org/gzip',
'https://ftp.gnu.org/gnu/gzip']
sources = [SOURCE_TAR_GZ]

# make sure the gzip, gunzip and compress binaries are available after installation
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.5-intel-2018a.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ description = "gzip (GNU zip) is a popular data compression program as a replace
toolchain = {'name': 'intel', 'version': '2018a'}

# eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz
source_urls = ['http://ftpmirror.gnu.org/gzip']
source_urls = ['http://ftpmirror.gnu.org/gzip',
'https://ftp.gnu.org/gnu/gzip']
sources = [SOURCE_TAR_GZ]

# make sure the gzip, gunzip and compress binaries are available after installation
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-GCC-4.9.2.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ description = "gzip (GNU zip) is a popular data compression program as a replace
toolchain = {'name': 'GCC', 'version': '4.9.2'}

# http://ftp.gnu.org/gnu/gzip/gzip-1.6.tar.gz
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS
sources = ['%(name)s-%(version)s.tar.gz']

# make sure the gzip, gunzip and compress binaries are available after installation
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
sources = ['%(name)s-%(version)s.tar.gz']

# download location for source files
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4-GCC-4.6.3.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ toolchain = {'name': 'GCC','version': '4.6.3'}
sources = ['%s-%s.tar.gz'%(name,version)]

# download location for source files
source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/v1.0/g/gzip/gzip-1.4.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ toolchain = SYSTEM
sources = [SOURCE_TAR_GZ]

# download location for source files
source_urls = ['https://ftpmirror.gnu.org/gnu/gzip']
source_urls = ['https://ftpmirror.gnu.org/gnu/gzip',
'https://ftp.gnu.org/gnu/gzip']

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-foss-2018a.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ description = "gzip (GNU zip) is a popular data compression program as a replace
toolchain = {'name': 'foss', 'version': '2018a'}

# eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz
source_urls = ['https://ftpmirror.gnu.org/gnu/gzip']
source_urls = ['https://ftpmirror.gnu.org/gnu/gzip',
'https://ftp.gnu.org/gnu/gzip']
sources = [SOURCE_TAR_GZ]

# make sure the gzip, gunzip and compress binaries are available after installation
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/v1.0/g/gzip/gzip-1.5-intel-2018a.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ description = "%(namelower)s (GNU zip) is a popular data compression program as
toolchain = {'name': 'intel', 'version': '2018a'}

# eg. http://ftp.gnu.org/gnu/gzip/gzip-1.5.tar.gz
source_urls = ['https://ftpmirror.gnu.org/gnu/gzip']
source_urls = ['https://ftpmirror.gnu.org/gnu/gzip',
'https://ftp.gnu.org/gnu/gzip']
sources = [SOURCE_TAR_GZ]

# make sure the gzip, gunzip and compress binaries are available after installation
Expand Down
3 changes: 2 additions & 1 deletion test/framework/easyconfigs/v2.0/doesnotexist.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ software_license_urls = ['http://localhost/my_license']
sources = ['%(name)s-%(version)s.tar.gz']

# download location for source files
source_urls = ['http://ftpmirror.gnu.org/gzip']
source_urls = ['http://ftpmirror.gnu.org/gzip',
'https://ftp.gnu.org/gnu/gzip']

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {'files': ["bin/gunzip", "bin/gzip"], 'dirs': []}
Expand Down
2 changes: 1 addition & 1 deletion test/framework/easyconfigs/v2.0/gzip.eb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ docurls = ["http://www.gnu.org/software/gzip/manual/gzip.html"]
software_license = GPLv3
software_license_urls = ['http://www.gnu.org/software/gzip/']

source_urls = [GNU_SOURCE]
source_urls = GNU_FTP_URLS
sources = [SOURCE_TAR_GZ]

sanity_check_paths = {
Expand Down
Loading