Skip to content

Commit ba7944b

Browse files
MaxDesiatovkateinoigakukun
MaxDesiatov
and
kateinoigakukun
authored
6.2: Build and install Testing and Foundation for Wasm Swift SDK (#83059)
Cherry-pick of #82285, merged as 05204bd **Explanation**: Include Swift Testing Foundation and libxml2 into the Swift SDK for WebAssembly **Scope**: Limited to Swift SDK for Wasm **Risk**: Low, due to limited scope. Changes were incubated on `main` for 2 weeks with no known issues. Swift Testing changes were incubated for more than a month. **Testing**: Manual testing with Swift SDKs produced off `main`. **Issue**: rdar://155865189 **Reviewer**: @MaxDesiatov --------- Co-authored-by: Yuta Saito <[email protected]>
1 parent 94f1efb commit ba7944b

File tree

1 file changed

+151
-41
lines changed

1 file changed

+151
-41
lines changed

‎utils/swift_build_support/swift_build_support/products/wasmswiftsdk.py

Lines changed: 151 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414

1515
from . import product
1616
from . import wasisysroot
17-
from .swift_testing import SwiftTestingCMakeShim
1817
from .wasmstdlib import WasmStdlib, WasmThreadsStdlib
18+
from .cmake_product import CMakeProduct
1919
from .. import shell
2020

2121

@@ -41,49 +41,156 @@ def should_test(self, host_target):
4141
def _target_package_path(self, swift_host_triple):
4242
return os.path.join(self.build_dir, 'Toolchains', swift_host_triple)
4343

44-
def _build_swift_testing(self, swift_host_triple, short_triple, wasi_sysroot):
45-
# TODO: We currently build swift-testing outside of SwiftTesting
46-
# build-script product because we build Wasm stdlib outside of
47-
# the main Swift build unit and we can't use build-script's cross
48-
# compilation infrastructure.
49-
# Once stdlib build is decoupled from compiler's CMake build unit
50-
# and we can use different CMake options for different targets
51-
# for stdlib build, we can fully unify library builds with the
52-
# regular path.
44+
def _append_platform_cmake_options(self, cmake_options, swift_host_triple, has_pthread, wasi_sysroot, extra_swift_flags):
45+
cmake_options.define('CMAKE_SYSTEM_NAME:STRING', 'WASI')
46+
cmake_options.define('CMAKE_SYSTEM_PROCESSOR:STRING', 'wasm32')
47+
cmake_options.define('CMAKE_C_COMPILER_TARGET', swift_host_triple)
48+
cmake_options.define('CMAKE_CXX_COMPILER_TARGET', swift_host_triple)
49+
cmake_options.define(
50+
'CMAKE_Swift_COMPILER_TARGET', swift_host_triple)
51+
cmake_options.define('CMAKE_SYSROOT', wasi_sysroot)
52+
5353
dest_dir = self._target_package_path(swift_host_triple)
54+
swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static')
55+
clang_resource_dir = os.path.join(swift_resource_dir, 'clang')
56+
57+
swift_flags = ['-sdk', wasi_sysroot, '-resource-dir',
58+
swift_resource_dir] + extra_swift_flags
59+
c_flags = ['-resource-dir', clang_resource_dir]
60+
cxx_flags = c_flags + ['-fno-exceptions']
61+
if has_pthread:
62+
clang_flags = ['-mthread-model', 'posix', '-pthread']
63+
c_flags.extend(clang_flags)
64+
cxx_flags.extend(clang_flags)
65+
swift_flags.extend(['-Xcc', '-matomics', '-Xcc', '-mbulk-memory',
66+
'-Xcc', '-mthread-model', '-Xcc', 'posix', '-Xcc', '-pthread'])
5467

55-
swift_testing = SwiftTestingCMakeShim(
68+
cmake_options.define('CMAKE_Swift_FLAGS', ' '.join(swift_flags))
69+
cmake_options.define('CMAKE_C_FLAGS', ' '.join(c_flags))
70+
cmake_options.define('CMAKE_CXX_FLAGS', ' '.join(cxx_flags))
71+
cmake_options.define('CMAKE_Swift_COMPILER_FORCED', 'TRUE')
72+
cmake_options.define('CMAKE_CXX_COMPILER_FORCED', 'TRUE')
73+
cmake_options.define('CMAKE_BUILD_TYPE', self.args.build_variant)
74+
75+
# Explicitly choose ar and ranlib from just-built LLVM tools since tools in the host system
76+
# unlikely support Wasm object format.
77+
native_toolchain_path = self.native_toolchain_path(self.args.host_target)
78+
cmake_options.define('CMAKE_AR', os.path.join(
79+
native_toolchain_path, 'bin', 'llvm-ar'))
80+
cmake_options.define('CMAKE_RANLIB', os.path.join(
81+
native_toolchain_path, 'bin', 'llvm-ranlib'))
82+
83+
def _build_libxml2(self, swift_host_triple, has_pthread, wasi_sysroot):
84+
libxml2 = CMakeProduct(
5685
args=self.args,
5786
toolchain=self.toolchain,
5887
source_dir=os.path.join(
59-
os.path.dirname(self.source_dir), 'swift-testing'),
60-
build_dir=os.path.join(
61-
os.path.dirname(self.build_dir),
62-
'swift-testing-%s' % short_triple))
88+
os.path.dirname(self.source_dir), 'libxml2'),
89+
build_dir=os.path.join(self.build_dir, 'libxml2', swift_host_triple))
90+
self._append_platform_cmake_options(
91+
libxml2.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, [])
92+
libxml2.cmake_options.define('LIBXML2_WITH_C14N', 'FALSE')
93+
libxml2.cmake_options.define('LIBXML2_WITH_CATALOG', 'FALSE')
94+
libxml2.cmake_options.define('LIBXML2_WITH_DEBUG', 'FALSE')
95+
libxml2.cmake_options.define('LIBXML2_WITH_DOCB', 'FALSE')
96+
libxml2.cmake_options.define('LIBXML2_WITH_FTP', 'FALSE')
97+
libxml2.cmake_options.define('LIBXML2_WITH_HTML', 'FALSE')
98+
libxml2.cmake_options.define('LIBXML2_WITH_HTTP', 'FALSE')
99+
libxml2.cmake_options.define('LIBXML2_WITH_ICONV', 'FALSE')
100+
libxml2.cmake_options.define('LIBXML2_WITH_ICU', 'FALSE')
101+
libxml2.cmake_options.define('LIBXML2_WITH_ISO8859X', 'FALSE')
102+
libxml2.cmake_options.define('LIBXML2_WITH_LEGACY', 'FALSE')
103+
libxml2.cmake_options.define('LIBXML2_WITH_LZMA', 'FALSE')
104+
libxml2.cmake_options.define('LIBXML2_WITH_MEM_DEBUG', 'FALSE')
105+
libxml2.cmake_options.define('LIBXML2_WITH_MODULES', 'FALSE')
106+
libxml2.cmake_options.define('LIBXML2_WITH_OUTPUT', 'TRUE')
107+
libxml2.cmake_options.define('LIBXML2_WITH_PATTERN', 'FALSE')
108+
libxml2.cmake_options.define('LIBXML2_WITH_PROGRAMS', 'FALSE')
109+
libxml2.cmake_options.define('LIBXML2_WITH_PUSH', 'TRUE')
110+
libxml2.cmake_options.define('LIBXML2_WITH_PYTHON', 'FALSE')
111+
libxml2.cmake_options.define('LIBXML2_WITH_READER', 'FALSE')
112+
libxml2.cmake_options.define('LIBXML2_WITH_REGEXPS', 'TRUE')
113+
libxml2.cmake_options.define('LIBXML2_WITH_RUN_DEBUG', 'FALSE')
114+
libxml2.cmake_options.define('LIBXML2_WITH_SAX1', 'FALSE')
115+
libxml2.cmake_options.define('LIBXML2_WITH_SCHEMAS', 'FALSE')
116+
libxml2.cmake_options.define('LIBXML2_WITH_SCHEMATRON', 'FALSE')
117+
libxml2.cmake_options.define('LIBXML2_WITH_TESTS', 'FALSE')
118+
libxml2.cmake_options.define('LIBXML2_WITH_TREE', 'TRUE')
119+
libxml2.cmake_options.define('LIBXML2_WITH_VALID', 'TRUE')
120+
libxml2.cmake_options.define('LIBXML2_WITH_WRITER', 'FALSE')
121+
libxml2.cmake_options.define('LIBXML2_WITH_XINCLUDE', 'FALSE')
122+
libxml2.cmake_options.define('LIBXML2_WITH_XPATH', 'TRUE')
123+
libxml2.cmake_options.define('LIBXML2_WITH_XPTR', 'FALSE')
124+
libxml2.cmake_options.define('LIBXML2_WITH_ZLIB', 'FALSE')
125+
libxml2.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE')
63126

64-
swift_testing.cmake_options.define('CMAKE_SYSTEM_NAME:STRING', 'WASI')
65-
swift_testing.cmake_options.define('CMAKE_SYSTEM_PROCESSOR:STRING', 'wasm32')
66-
swift_testing.cmake_options.define(
67-
'CMAKE_CXX_COMPILER_TARGET', swift_host_triple)
68-
swift_testing.cmake_options.define(
69-
'CMAKE_Swift_COMPILER_TARGET', swift_host_triple)
70-
swift_testing.cmake_options.define('CMAKE_SYSROOT', wasi_sysroot)
71-
swift_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'swift_static')
72-
swift_testing.cmake_options.define(
73-
'CMAKE_Swift_FLAGS',
74-
f'-sdk {wasi_sysroot} -resource-dir {swift_resource_dir}')
75-
clang_resource_dir = os.path.join(dest_dir, 'usr', 'lib', 'clang')
127+
cmake_thread_enabled = 'TRUE' if has_pthread else 'FALSE'
128+
libxml2.cmake_options.define('LIBXML2_WITH_THREAD_ALLOC', cmake_thread_enabled)
129+
libxml2.cmake_options.define('LIBXML2_WITH_THREADS', cmake_thread_enabled)
130+
libxml2.cmake_options.define('HAVE_PTHREAD_H', cmake_thread_enabled)
131+
132+
libxml2.build_with_cmake([], self.args.build_variant, [],
133+
prefer_native_toolchain=True)
134+
with shell.pushd(libxml2.build_dir):
135+
shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/', '--component', 'development'],
136+
env={'DESTDIR': wasi_sysroot})
137+
138+
def _build_foundation(self, swift_host_triple, has_pthread, wasi_sysroot):
139+
source_root = os.path.dirname(self.source_dir)
140+
host_toolchain = self.native_toolchain_path(self.args.host_target)
141+
142+
foundation = CMakeProduct(
143+
args=self.args,
144+
toolchain=self.toolchain,
145+
source_dir=os.path.join(
146+
os.path.dirname(self.source_dir), 'swift-corelibs-foundation'),
147+
build_dir=os.path.join(self.build_dir, 'foundation', swift_host_triple))
148+
self._append_platform_cmake_options(
149+
foundation.cmake_options, swift_host_triple, has_pthread, wasi_sysroot, [])
150+
foundation.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE')
151+
foundation.cmake_options.define('FOUNDATION_BUILD_TOOLS', 'FALSE')
152+
foundation.cmake_options.define('_SwiftCollections_SourceDIR', os.path.join(source_root, 'swift-collections'))
153+
foundation.cmake_options.define('_SwiftFoundation_SourceDIR', os.path.join(source_root, 'swift-foundation'))
154+
foundation.cmake_options.define('_SwiftFoundationICU_SourceDIR', os.path.join(source_root, 'swift-foundation-icu'))
155+
foundation.cmake_options.define('SwiftFoundation_MACRO', os.path.join(host_toolchain, 'lib', 'swift', 'host', 'plugins'))
156+
157+
foundation.cmake_options.define('LIBXML2_INCLUDE_DIR', os.path.join(wasi_sysroot, 'include', 'libxml2'))
158+
foundation.cmake_options.define('LIBXML2_LIBRARY', os.path.join(wasi_sysroot, 'lib'))
159+
160+
foundation.build_with_cmake([], self.args.build_variant, [],
161+
prefer_native_toolchain=True)
162+
163+
dest_dir = self._target_package_path(swift_host_triple)
164+
with shell.pushd(foundation.build_dir):
165+
shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/usr'],
166+
env={'DESTDIR': dest_dir})
167+
168+
def _build_swift_testing(self, swift_host_triple, has_pthread, wasi_sysroot):
169+
swift_testing = CMakeProduct(
170+
args=self.args,
171+
toolchain=self.toolchain,
172+
source_dir=os.path.join(
173+
os.path.dirname(self.source_dir), 'swift-testing'),
174+
build_dir=os.path.join(self.build_dir, 'swift-testing', swift_host_triple))
175+
# For statically linked objects in an archive, we have to use singlethreaded
176+
# LLVM codegen unit to prevent runtime metadata sections from being stripped
177+
# at link-time.
178+
self._append_platform_cmake_options(
179+
swift_testing.cmake_options, swift_host_triple, has_pthread, wasi_sysroot,
180+
extra_swift_flags=['-Xfrontend', '-enable-single-module-llvm-emission'])
181+
swift_testing.cmake_options.define('BUILD_SHARED_LIBS', 'FALSE')
76182
swift_testing.cmake_options.define(
77-
'CMAKE_CXX_FLAGS', f'-resource-dir {clang_resource_dir}')
78-
swift_testing.cmake_options.define('CMAKE_Swift_COMPILER_FORCED', 'TRUE')
79-
swift_testing.cmake_options.define('CMAKE_CXX_COMPILER_FORCED', 'TRUE')
183+
'CMAKE_Swift_COMPILATION_MODE', 'wholemodule')
184+
swift_testing.cmake_options.define('SwiftTesting_MACRO', 'NO')
80185

81-
swift_testing.build('wasi-wasm32')
186+
swift_testing.build_with_cmake([], self.args.build_variant, [],
187+
prefer_native_toolchain=True)
188+
dest_dir = self._target_package_path(swift_host_triple)
82189
with shell.pushd(swift_testing.build_dir):
83190
shell.call([self.toolchain.cmake, '--install', '.', '--prefix', '/usr'],
84191
env={'DESTDIR': dest_dir})
85192

86-
def _build_target_package(self, swift_host_triple, short_triple,
193+
def _build_target_package(self, swift_host_triple, has_pthread,
87194
stdlib_build_path, llvm_runtime_libs_build_path,
88195
wasi_sysroot):
89196

@@ -103,8 +210,11 @@ def _build_target_package(self, swift_host_triple, short_triple,
103210
shell.call([self.toolchain.cmake, '--install', '.',
104211
'--component', 'clang_rt.builtins-wasm32'],
105212
env={'DESTDIR': clang_dir})
213+
214+
self._build_libxml2(swift_host_triple, has_pthread, wasi_sysroot)
215+
self._build_foundation(swift_host_triple, has_pthread, wasi_sysroot)
106216
# Build swift-testing
107-
self._build_swift_testing(swift_host_triple, short_triple, wasi_sysroot)
217+
self._build_swift_testing(swift_host_triple, has_pthread, wasi_sysroot)
108218

109219
return dest_dir
110220

@@ -114,26 +224,26 @@ def build(self, host_target):
114224
build_root, '%s-%s' % ('wasmllvmruntimelibs', host_target))
115225

116226
target_packages = []
117-
# NOTE: We have three types of target triples:
227+
# NOTE: We have two types of target triples:
118228
# 1. swift_host_triple: The triple used by the Swift compiler's '-target' option
119229
# 2. clang_multiarch_triple: The triple used by Clang to find library
120230
# and header paths from the sysroot
121231
# https://github.com/llvm/llvm-project/blob/73ef397fcba35b7b4239c00bf3e0b4e689ca0add/clang/lib/Driver/ToolChains/WebAssembly.cpp#L29-L36
122-
# 3. short_triple: The triple used by build-script to name the build directory
123-
for swift_host_triple, clang_multiarch_triple, short_triple, build_basename in [
124-
('wasm32-unknown-wasi', 'wasm32-wasi', 'wasi-wasm32', 'wasmstdlib'),
125-
# TODO: Enable threads stdlib once sdk-generator supports multi-target SDK
232+
for swift_host_triple, clang_multiarch_triple, build_basename, build_sdk, has_pthread in [
233+
('wasm32-unknown-wasi', 'wasm32-wasi', 'wasmstdlib', True, False),
234+
# TODO: Include p1-threads in the Swift SDK once sdk-generator supports multi-target SDK
126235
# ('wasm32-unknown-wasip1-threads', 'wasm32-wasip1-threads',
127-
# 'wasip1-threads-wasm32', 'wasmthreadsstdlib'),
236+
# 'wasmthreadsstdlib', False, True),
128237
]:
129238
stdlib_build_path = os.path.join(
130239
build_root, '%s-%s' % (build_basename, host_target))
131240
wasi_sysroot = wasisysroot.WASILibc.sysroot_install_path(
132241
build_root, clang_multiarch_triple)
133242
package_path = self._build_target_package(
134-
swift_host_triple, short_triple, stdlib_build_path,
243+
swift_host_triple, has_pthread, stdlib_build_path,
135244
llvm_runtime_libs_build_path, wasi_sysroot)
136-
target_packages.append((swift_host_triple, wasi_sysroot, package_path))
245+
if build_sdk:
246+
target_packages.append((swift_host_triple, wasi_sysroot, package_path))
137247

138248
swiftc_path = os.path.abspath(self.toolchain.swiftc)
139249
toolchain_path = os.path.dirname(os.path.dirname(swiftc_path))

0 commit comments

Comments
 (0)