Prefer multiprocessing.cpu_count

This takes into account thread affinity, at least on python 3.13 and
above.

Author: sbc100

test/common.py

@@ -1844,7 +1844,7 @@ def get_library(self, name, generated_libs, configure=['sh', './configure'],  #
     if env_init is None:
       env_init = {}
     if make_args is None:
-      make_args = ['-j', str(shared.get_num_cores())]
+      make_args = ['-j', str(utils.get_num_cores())]
 
     build_dir = self.get_build_dir()
 

test/parallel_testsuite.py

@@ -431,4 +431,4 @@ def __init__(self, co):
 def num_cores():
   if NUM_CORES:
     return int(NUM_CORES)
-  return multiprocessing.cpu_count()
+  return utils.get_num_cores()

tools/js_optimizer.py

@@ -244,7 +244,7 @@ def check_symbol_mapping(p):
     # if we are making source maps, we want our debug numbering to start from the
     # top of the file, so avoid breaking the JS into chunks
 
-    intended_num_chunks = round(shared.get_num_cores() * NUM_CHUNKS_PER_CORE)
+    intended_num_chunks = round(utils.get_num_cores() * NUM_CHUNKS_PER_CORE)
     chunk_size = min(MAX_CHUNK_SIZE, max(MIN_CHUNK_SIZE, total_size / intended_num_chunks))
     chunks = chunkify(funcs, chunk_size)
 

tools/shared.py

@@ -115,10 +115,6 @@ def run_process(cmd, check=True, input=None, *args, **kw):
   return ret
 
 
-def get_num_cores():
-  return int(os.environ.get('EMCC_CORES', os.cpu_count()))
-
-
 def returncode_to_str(code):
   assert code != 0
   if code < 0:
@@ -180,7 +176,7 @@ def get_finished_process():
       except subprocess.TimeoutExpired:
         pass
 
-  num_parallel_processes = get_num_cores()
+  num_parallel_processes = utils.get_num_cores()
   temp_files = get_temp_files()
   i = 0
   num_completed = 0

tools/system_libs.py

@@ -159,7 +159,7 @@ def get_top_level_ninja_file():
 
 
 def run_ninja(build_dir):
-  cmd = ['ninja', '-C', build_dir, f'-j{shared.get_num_cores()}']
+  cmd = ['ninja', '-C', build_dir, f'-j{utils.get_num_cores()}']
   if shared.PRINT_SUBPROCS:
     cmd.append('-v')
   shared.check_call(cmd, env=clean_env())
@@ -538,7 +538,7 @@ def build_objects(self, build_dir):
       # Choose a chunk size that is large enough to avoid too many subprocesses
       # but not too large to avoid task starvation.
       # For now the heuristic is to split inputs by 2x number of cores.
-      chunk_size = max(1, len(objects) // (2 * shared.get_num_cores()))
+      chunk_size = max(1, len(objects) // (2 * utils.get_num_cores()))
       # Convert batches to commands.
       for cmd, srcs in batches.items():
         cmd = list(cmd)

tools/utils.py

@@ -3,6 +3,7 @@
 # University of Illinois/NCSA Open Source License.  Both these licenses can be
 # found in the LICENSE file.
 
+import multiprocessing
 import os
 import shutil
 import sys
@@ -112,6 +113,14 @@ def delete_contents(dirname, exclude=None):
       delete_file(entry)
 
 
+def get_num_cores():
+  # Avoid os.cpu_count() here because it doesn't take into account thread affinity.
+  # We could use os.process_cpu_count() but its only available in 3.13.
+  # multiprocessing.cpu_count() will use os.process_cpu_count() automatically
+  # on 3.13 and above
+  return int(os.environ.get('EMCC_CORES', multiprocessing.cpu_count()))
+
+
 # TODO(sbc): Replace with functools.cache, once we update to python 3.9
 memoize = functools.lru_cache(maxsize=None)