fix(loader): Load files concurrently with a limit to avoid "too many open files" errors

Author: kristijanhusak

lua/orgmode/files/init.lua

@@ -39,16 +39,14 @@ function OrgFiles:load(force)
   end
 
   self.load_state = 'loading'
-  local actions = vim.tbl_map(function(filename)
+  return Promise.map(function(filename)
     return self:load_file(filename):next(function(orgfile)
       if orgfile then
         self.files[orgfile.filename] = orgfile
       end
       return orgfile
     end)
-  end, self:_files(true))
-
-  return Promise.all(actions):next(function()
+  end, self:_files(true), 50):next(function()
     self.load_state = 'loaded'
     return self
   end)

lua/orgmode/utils/promise.lua

@@ -332,6 +332,53 @@ function Promise.all(list)
   end)
 end
 
+--- Equivalents to JavaScript's Promise.map with concurrency limit.
+--- @param callback fun(value: any, index: number, array: any[]): any
+--- @param list any[]: promise or non-promise values
+--- @param concurrency? number: limit number of concurrent items processing
+--- @return OrgPromise
+function Promise.map(callback, list, concurrency)
+  vim.validate({
+    list = { list, 'table' },
+    callback = { callback, 'function' },
+    concurrency = { concurrency, 'number', true },
+  })
+
+  local results = {}
+  local processing = 0
+  local index = 1
+  concurrency = concurrency or #list
+
+  return Promise.new(function(resolve, reject)
+    local function processNext()
+      if index > #list then
+        if processing == 0 then
+          resolve(results)
+        end
+        return
+      end
+
+      local i = index
+      index = index + 1
+      processing = processing + 1
+
+      Promise.resolve(callback(list[i], i, list))
+        :next(function(...)
+          results[i] = ...
+          processing = processing - 1
+          processNext()
+        end)
+        :catch(function(...)
+          reject(...)
+        end)
+    end
+
+    for _ = 1, concurrency do
+      processNext()
+    end
+  end)
+end
+
 --- Equivalents to JavaScript's Promise.race.
 --- @param list any[]: promise or non-promise values
 --- @return OrgPromise