Sourcemap improvements

docs/emcc.txt

@@ -181,15 +181,19 @@ Options that are modified or new in *emcc* are listed below:
    alongside the wasm object files. This option must be used together
    with "-c".
 
-"-gsource-map"
+"-gsource-map[=inline]"
    [link] Generate a source map using LLVM debug information (which
    must be present in object files, i.e., they should have been
    compiled with "-g"). When this option is provided, the **.wasm**
    file is updated to have a "sourceMappingURL" section. The resulting
    URL will have format: "<base-url>" + "<wasm-file-name>" + ".map".
    "<base-url>" defaults to being empty (which means the source map is
    served from the same directory as the Wasm file). It can be changed
-   using --source-map-base.
+   using --source-map-base. It is possbile to apply path substitutions
+   to the referenced sources using the "-sSOURCE_MAP_PREFIXES" option.
+   If "inline" is specified, the sources content is embedded in the
+   source map (in this case you don't need path substitution, but it
+   comes with the cost of having a large source map file).
 
 "-g<level>"
    [compile+link] Controls the level of debuggability. Each level

emcc.py

@@ -1196,8 +1196,8 @@ def consume_arg_file():
           else:
             settings.SEPARATE_DWARF = True
           settings.GENERATE_DWARF = 1
-        elif requested_level == 'source-map':
-          settings.GENERATE_SOURCE_MAP = 1
+        elif requested_level in ['source-map', 'source-map=inline']:
+          settings.GENERATE_SOURCE_MAP = 1 if requested_level == 'source-map' else 2
           settings.EMIT_NAME_SECTION = 1
           newargs[i] = '-g'
         else:

site/source/docs/tools_reference/emcc.rst

@@ -172,16 +172,23 @@ Options that are modified or new in *emcc* are listed below:
 
 .. _emcc-gsource-map:
 
-``-gsource-map``
+``-gsource-map[=inline]``
   [link]
   Generate a source map using LLVM debug information (which must
   be present in object files, i.e., they should have been compiled with ``-g``).
+
   When this option is provided, the **.wasm** file is updated to have a
   ``sourceMappingURL`` section. The resulting URL will have format:
   ``<base-url>`` + ``<wasm-file-name>`` + ``.map``. ``<base-url>`` defaults
   to being empty (which means the source map is served from the same directory
   as the Wasm file). It can be changed using :ref:`--source-map-base <emcc-source-map-base>`.
 
+  Path substitution can be applied to the referenced sources using the
+  ``-sSOURCE_MAP_PREFIXES`` (:ref:`link <source_map_prefixes>`).
+  If ``inline`` is specified, the sources content is embedded in the source map
+  (in this case you don't need path substitution, but it comes with the cost of
+  having a large source map file).
+
 .. _emcc-gN:
 
 ``-g<level>``

site/source/docs/tools_reference/settings_reference.rst

@@ -3130,6 +3130,21 @@ This is enabled automatically when using -gsource-map with sanitizers.
 
 Default value: false
 
+.. _source_map_prefixes:
+
+SOURCE_MAP_PREFIXES
+===================
+
+List of path substitutions to apply in the "sources" field of the source map.
+Corresponds to the ``--prefix`` option used in ``tools/wasm-sourcemap.py``.
+Must be used with ``-gsource-map``.
+
+This setting allows to map path prefixes to the proper ones so that the final
+(possibly relative) URLs point to the correct locations :
+``-sSOURCE_MAP_PREFIXES=['/old/path=/new/path']``
+
+Default value: []
+
 .. _default_to_cxx:
 
 DEFAULT_TO_CXX

src/settings.js

@@ -2045,6 +2045,17 @@ var USE_OFFSET_CONVERTER = false;
 // This is enabled automatically when using -gsource-map with sanitizers.
 var LOAD_SOURCE_MAP = false;
 
+// List of path substitutions to apply in the "sources" field of the source map.
+// Corresponds to the ``--prefix`` option used in ``tools/wasm-sourcemap.py``.
+// Must be used with ``-gsource-map``.
+//
+// This setting allows to map path prefixes to the proper ones so that the final
+// (possibly relative) URLs point to the correct locations :
+// ``-sSOURCE_MAP_PREFIXES=['/old/path=/new/path']``
+//
+// [link]
+var SOURCE_MAP_PREFIXES = [];
+
 // Default to c++ mode even when run as ``emcc`` rather then ``emc++``.
 // When this is disabled ``em++`` is required linking C++ programs. Disabling
 // this will match the behaviour of gcc/g++ and clang/clang++.

src/settings_internal.js

@@ -204,7 +204,10 @@ var USE_READY_PROMISE = true;
 // If true, building against Emscripten's wasm heap memory profiler.
 var MEMORYPROFILER = false;
 
-var GENERATE_SOURCE_MAP = false;
+// Set automatically to :
+// - 1 when using `-gsource-map`
+// - 2 when using `gsource-map=inline` (embed sources content in souce map)
+var GENERATE_SOURCE_MAP = 0;
 
 var GENERATE_DWARF = false;
 

test/test_other.py

@@ -50,6 +50,7 @@
 import line_endings
 from tools import webassembly
 from tools.settings import settings
+from tools.system_libs import DETERMINISTIC_PREFIX
 
 scons_path = shutil.which('scons')
 emmake = shared.bat_suffix(path_from_root('emmake'))
@@ -10350,25 +10351,84 @@ def test_check_sourcemapurl_default(self, *args):
     source_mapping_url_content = webassembly.to_leb(len('sourceMappingURL')) + b'sourceMappingURL' + webassembly.to_leb(len('a.wasm.map')) + b'a.wasm.map'
     self.assertIn(source_mapping_url_content, output)
 
-  def test_wasm_sourcemap(self):
-    # The no_main.c will be read (from relative location) due to speficied "-s"
+  @parameterized({
+    '': ([], [], []),
+    'prefix_wildcard': ([], ['--prefix', '=wasm-src://'], []),
+    'prefix_partial': ([], ['--prefix', '/emscripten/=wasm-src:///emscripten/'], []),
+    'sources': (['--sources'], [], ['--load-prefix', '/emscripten/test/other/wasm_sourcemap=.'])
+  })
+  @parameterized({
+    '': ('/',),
+    'basepath': ('/emscripten/test',)
+  })
+  def test_wasm_sourcemap(self, sources, prefix, load_prefix, basepath):
+    # The no_main.c will be read from relative location if necessary (depends
+    # on --sources and --load-prefix options).
     shutil.copy(test_file('other/wasm_sourcemap/no_main.c'), '.')
+    DW_AT_decl_file = '/emscripten/test/other/wasm_sourcemap/no_main.c'
     wasm_map_cmd = [PYTHON, path_from_root('tools/wasm-sourcemap.py'),
-                    '--sources', '--prefix', '=wasm-src://',
-                    '--load-prefix', '/emscripten/test/other/wasm_sourcemap=.',
+                    *sources, *prefix, *load_prefix,
                     '--dwarfdump-output',
                     test_file('other/wasm_sourcemap/foo.wasm.dump'),
                     '-o', 'a.out.wasm.map',
                     test_file('other/wasm_sourcemap/foo.wasm'),
-                    '--basepath=' + os.getcwd()]
+                    '--basepath=' + basepath]
     self.run_process(wasm_map_cmd)
     output = read_file('a.out.wasm.map')
-    # has "sources" entry with file (includes also `--prefix =wasm-src:///` replacement)
-    self.assertIn('wasm-src:///emscripten/test/other/wasm_sourcemap/no_main.c', output)
-    # has "sourcesContent" entry with source code (included with `-s` option)
-    self.assertIn('int foo()', output)
-    # has some entries
-    self.assertRegex(output, r'"mappings":\s*"[A-Za-z0-9+/]')
+    # "sourcesContent" contains source code iff --sources is specified.
+    self.assertIn('int foo()' if sources else '"sourcesContent":[]', output)
+    if prefix: # "sources" contains URL with prefix path substition if provided
+      sources_url = 'wasm-src:///emscripten/test/other/wasm_sourcemap/no_main.c'
+    else: # otherwise a path relative to the given basepath.
+      sources_url = utils.normalize_path(os.path.relpath(DW_AT_decl_file, basepath))
+    self.assertIn(sources_url, output)
+    # "mappings" contains valid Base64 VLQ segments.
+    self.assertRegex(output, r'"mappings":\s*"(?:[A-Za-z0-9+\/]+[,;]?)+"')
+
+  @parameterized({
+    '': ([], 0),
+    'prefix': ([
+      '<cwd>=file:///path/to/src',
+      DETERMINISTIC_PREFIX + '=file:///path/to/emscripten',
+    ], 0),
+    'sources': ([], 1)
+  })
+  def test_emcc_sourcemap_options(self, prefixes, sources):
+    wasm_sourcemap = importlib.import_module('tools.wasm-sourcemap')
+    cwd = os.getcwd()
+    src_file = shutil.copy(test_file('hello_123.c'), cwd)
+    lib_file = DETERMINISTIC_PREFIX + '/system/lib/libc/musl/src/stdio/fflush.c'
+    if prefixes:
+      prefixes = [p.replace('<cwd>', cwd) for p in prefixes]
+    self.set_setting('SOURCE_MAP_PREFIXES', prefixes)
+    args = ['-gsource-map=inline' if sources else '-gsource-map']
+    self.emcc(src_file, args=args, output_filename='test.js')
+    output = read_file('test.wasm.map')
+    # Check source file resolution
+    p = wasm_sourcemap.Prefixes(prefixes, base_path=cwd)
+    self.assertEqual(len(p.prefixes), len(prefixes))
+    src_file_url = p.resolve(utils.normalize_path(src_file))
+    lib_file_url = p.resolve(utils.normalize_path(lib_file))
+    if prefixes:
+      self.assertEqual(src_file_url, 'file:///path/to/src/hello_123.c')
+      self.assertEqual(lib_file_url, 'file:///path/to/emscripten/system/lib/libc/musl/src/stdio/fflush.c')
+    else:
+      self.assertEqual(src_file_url, 'hello_123.c')
+      self.assertEqual(lib_file_url, '/emsdk/emscripten/system/lib/libc/musl/src/stdio/fflush.c')
+    # "sources" contains resolved filepath.
+    self.assertIn(f'"{src_file_url}"', output)
+    self.assertIn(f'"{lib_file_url}"', output)
+    # "sourcesContent" contains source code iff -gsource-map=inline is specified.
+    if sources:
+      p = wasm_sourcemap.Prefixes(prefixes, preserve_deterministic_prefix=False)
+      for filepath in [src_file, lib_file]:
+        resolved_path = p.resolve(utils.normalize_path(filepath))
+        sources_content = json.dumps(read_file(resolved_path))
+        self.assertIn(sources_content, output)
+    else:
+      self.assertIn('"sourcesContent":[]', output)
+    # "mappings" contains valid Base64 VLQ segments.
+    self.assertRegex(output, r'"mappings":\s*"(?:[A-Za-z0-9+\/]+[,;]?)+"')
 
   def test_wasm_sourcemap_dead(self):
     wasm_map_cmd = [PYTHON, path_from_root('tools/wasm-sourcemap.py'),

tools/building.py

@@ -1131,6 +1131,13 @@ def emit_wasm_source_map(wasm_file, map_file, final_wasm):
                    '--dwarfdump=' + LLVM_DWARFDUMP,
                    '-o',  map_file,
                    '--basepath=' + base_path]
+
+  if settings.SOURCE_MAP_PREFIXES:
+    sourcemap_cmd += ['--prefix', *settings.SOURCE_MAP_PREFIXES]
+
+  if settings.GENERATE_SOURCE_MAP == 2:
+    sourcemap_cmd += ['--sources']
+
   check_call(sourcemap_cmd)
 
 

tools/system_libs.py

@@ -44,7 +44,7 @@
 
 # A (fake) deterministic emscripten path to use in __FILE__ macro and debug info
 # to produce reproducible builds across platforms.
-DETERMINISITIC_PREFIX = '/emsdk/emscripten'
+DETERMINISTIC_PREFIX = '/emsdk/emscripten'
 
 
 def files_in_path(path, filenames):
@@ -588,9 +588,9 @@ def get_cflags(self):
 
     source_dir = utils.path_from_root()
     relative_source_dir = os.path.relpath(source_dir, self.build_dir)
-    cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISITIC_PREFIX}',
-               f'-ffile-prefix-map={relative_source_dir}={DETERMINISITIC_PREFIX}',
-               f'-fdebug-compilation-dir={DETERMINISITIC_PREFIX}']
+    cflags += [f'-ffile-prefix-map={source_dir}={DETERMINISTIC_PREFIX}',
+               f'-ffile-prefix-map={relative_source_dir}={DETERMINISTIC_PREFIX}',
+               f'-fdebug-compilation-dir={DETERMINISTIC_PREFIX}']
     return cflags
 
   def get_base_name_prefix(self):