Fix job file configuration

Author: wholebuzz

docs/modules/mapreduce.md

@@ -28,7 +28,7 @@
 
 **Returns:** <Key, Value\>(`mapper`: [*Mapper*](../interfaces/types.mapper.md)<Key, Value\>, `combiner`: [*Reducer*](../interfaces/types.reducer.md)<Key, Value\> \| *undefined*, `args`: [*MapReduceRuntimeConfig*](../interfaces/types.mapreduceruntimeconfig.md)<Key, Value\>, `options`: DatabaseCopyOptions) => *Promise*<void\>
 
-Defined in: [src/mapreduce.ts:201](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L201)
+Defined in: [src/mapreduce.ts:204](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L204)
 
 ___
 
@@ -45,7 +45,7 @@ ___
 
 **Returns:** *Promise*<[*InputSplit*](../interfaces/config.inputsplit.md)[]\>
 
-Defined in: [src/mapreduce.ts:346](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L346)
+Defined in: [src/mapreduce.ts:349](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L349)
 
 ___
 
@@ -98,7 +98,7 @@ ___
 
 **Returns:** *Promise*<void\>
 
-Defined in: [src/mapreduce.ts:278](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L278)
+Defined in: [src/mapreduce.ts:281](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L281)
 
 ___
 
@@ -124,7 +124,7 @@ ___
 
 **Returns:** *Promise*<void\>
 
-Defined in: [src/mapreduce.ts:238](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L238)
+Defined in: [src/mapreduce.ts:241](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L241)
 
 ___
 
@@ -149,4 +149,4 @@ ___
 
 **Returns:** *Promise*<void\>
 
-Defined in: [src/mapreduce.ts:212](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L212)
+Defined in: [src/mapreduce.ts:215](https://github.com/wholebuzz/mapreduce/blob/master/src/mapreduce.ts#L215)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wholebuzz/mapreduce",
-  "version": "0.0.18",
+  "version": "0.0.19",
   "description": "Communication-free MapReduce for the 99%",
   "keywords": [
     "mapreduce"

src/cli.js

@@ -282,7 +282,15 @@ async function main() {
           throw new Error('No outputPath')
         }
         try {
-          await mapReduce(await prepareRuntime(fileSystem, logger, { ...options, configuration }))
+          await mapReduce(
+            await prepareRuntime(fileSystem, logger, {
+              ...options,
+              configuration: {
+                ...options.configuration,
+                ...configuration,
+              },
+            })
+          )
           returnValue = { inputPaths: args.inputPaths, outputPath: args.outputPath }
         } catch (err) {
           logger.info(err.message)

src/mapreduce.ts

@@ -47,6 +47,9 @@ export async function mapReduce<Key, Value>(args: MapReduceRuntimeConfig<Key, Va
   if (!shuffleDirectory.endsWith('/')) throw new Error('shuffleDirectory should end with slash')
   if (args.logger) {
     args.logger.info(`mapReduce configuration ${JSON.stringify(args.configuration ?? {})}`)
+    if ((args.inputShardFilter || args.outputShardFilter) && !args.shuffleDirectory) {
+      args.logger.info(`WARNING: no shuffleDirectory specified`)
+    }
   }
 
   // Since local files show partial writes they need extra synchronization
@@ -87,7 +90,7 @@ export async function mapReduce<Key, Value>(args: MapReduceRuntimeConfig<Key, Va
             },
           ],
       inputShardIndex: inputSplit,
-      orderBy: args.inputShardBy ? [ args.inputShardBy ] : undefined,
+      orderBy: args.inputShardBy ? [args.inputShardBy] : undefined,
       // dir/shuffle-SSSS-of-NNNN.inputshard-0000-of-0004.jsonl.gz
       outputFile:
         (args.unpatchReduce ? args.outputPath : shuffleDirectory + `${shuffleFilenameFormat}`) +

src/plugins.ts

@@ -167,4 +167,4 @@ export function getSubPropertySetter(path: string): (_: Record<string, any>, val
   } else {
     return (x, value) => setSubProperty(x, nested, value)
   }
-}
+}