diff --git a/Makefile b/Makefile index 9af3667c1a..f71128bef0 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,9 @@ dev: make -C docs @echo DONE -# Keystroke-saver -itso: build check install +# Keystroke-savers +it: build check +so: install # Please see comments in ./create-release-tarball as well as # https://miller.readthedocs.io/en/latest/build/#creating-a-new-release-for-developers diff --git a/docs/src/manpage.md b/docs/src/manpage.md index 4a495d0ab5..1a0a113e06 100644 --- a/docs/src/manpage.md +++ b/docs/src/manpage.md @@ -2979,5 +2979,5 @@ SEE ALSO - 2021-11-23 MILLER(1) + 2021-11-24 MILLER(1) diff --git a/docs/src/manpage.txt b/docs/src/manpage.txt index 4afa7a93a3..1e1addc4b0 100644 --- a/docs/src/manpage.txt +++ b/docs/src/manpage.txt @@ -2958,4 +2958,4 @@ SEE ALSO - 2021-11-23 MILLER(1) + 2021-11-24 MILLER(1) diff --git a/internal/pkg/auxents/repl/entry.go b/internal/pkg/auxents/repl/entry.go index e8867af4bf..de57632ad2 100644 --- a/internal/pkg/auxents/repl/entry.go +++ b/internal/pkg/auxents/repl/entry.go @@ -137,7 +137,7 @@ func ReplMain(args []string) int { argi += 1 } - } else if cli.FLAG_TABLE.Parse(args, argc, &argi, &options) { + } else if cli.FLAG_TABLE.Parse(args, argc, &argi, options) { } else { replUsage(replName, os.Stderr, 1) @@ -150,7 +150,7 @@ func ReplMain(args []string) int { // --auto-flatten is on by default. But if input and output formats are both JSON, // then we don't need to actually do anything. See also mlrcli_parse.go. options.WriterOptions.AutoFlatten = cli.DecideFinalFlatten(&options.WriterOptions) - options.WriterOptions.AutoUnflatten = cli.DecideFinalUnflatten(&options) + options.WriterOptions.AutoUnflatten = cli.DecideFinalUnflatten(options) repl, err := NewRepl( exeName, @@ -159,7 +159,7 @@ func ReplMain(args []string) int { showPrompts, astPrintMode, doWarnings, - &options, + options, ) if err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/internal/pkg/cli/option_types.go b/internal/pkg/cli/option_types.go index 1cd1e98cd6..c10c62fa85 100644 --- a/internal/pkg/cli/option_types.go +++ b/internal/pkg/cli/option_types.go @@ -157,8 +157,8 @@ type TOptions struct { } // ---------------------------------------------------------------- -func DefaultOptions() TOptions { - return TOptions{ +func DefaultOptions() *TOptions { + return &TOptions{ ReaderOptions: DefaultReaderOptions(), WriterOptions: DefaultWriterOptions(), diff --git a/internal/pkg/climain/mlrcli_parse.go b/internal/pkg/climain/mlrcli_parse.go index dd2f725eff..0db3692acf 100644 --- a/internal/pkg/climain/mlrcli_parse.go +++ b/internal/pkg/climain/mlrcli_parse.go @@ -1,3 +1,71 @@ +// Miller main command-line parsing. +// +// Before Miller 6 the ordering was: +// * mlr +// * main flags like --icsv --ojson +// * verbs and their flags like cat -n +// * data-file names +// and the command-line parser was one-pass. +// +// In Miller 6 we have as keystroke-reducers 'mlr -s', for '#!mlr -s', +// or simply better support for mlr inside of '#!/bin/sh' scripts: +// +// mlr {flags} {verbs} -- [more flags] [more verbs] {data file names} +// [the part inside a script file] [the part outside] +// +// For example, suppose someone wants to reuse the following: +// mlr --icsv --json head -n 10 +// either via a #!mlr -s script, maybe "peek.mlr": +// #!/usr/bin/env mlr -s +// --icsv --json head -n 10 +// or a #!/bin/bash script, maybe "peek.sh" +// #!/bin/bash +// mlr --icsv --json head -n 10 -- "$@" +// Then they can do 'peek.mlr myfile.csv' or 'peek.sh myfile.csv' which is great. +// +// But suppose they want to do +// peek.sh --jlistwrap myfile.csv +// Then the Miller command line received here is +// mlr --icsv --json head -n 10 -- --jlistwrap myfile.csv +// Or, maybe their part inside the '#!mlr' or '#!/bin/sh' file is all verbs, +// and they want to specify format-flags like '--icsv --ojson' outside of that +// script. It's very reasonable for them to want to put the --jlistwrap, +// --icsv, --ojson, etc. after their keystroke-saver script. But this now means +// that there can be main-flags (and/or 'then someotherverb') *after* the verb +// chain from inside the keystroke-saver. +// +// Also, verbs/transformers must be constructed *after* all main-flags are +// parsed -- since some of them depend on main-flags, e.g. join, put/filter, +// and tee which use things like --csv for their I/O options. +// +// Therefore the command-line parsing is now two-pass. +// * Pass 1: +// o 'mlr' is first +// o Split the []args into "sequences" of main-flags, verbs and their flags, +// and data-file names. +// o For example in the above 'mlr --icsv --json head -n 10 -- --jlistwrap myfile.csv' +// we have +// main-flag sequences ['--icsv'] ['--json'] [--jlistwrap], +// verb-seqeunce ['head' '-n' '10'] +// data-file names ['myfile.csv']. +// o Any exiting flags like --version or --help are dispatched here. +// o To do that splitting we invoke the flag-table parser with throwaway options struct, +// and we invoke the transformers' ParseCLI functions with doConstruct = false. +// * Pass 2: +// o Process the flag-sequences in the order they were encountered, into a +// for-real-use options struct. +// o Process the verb-sequences in the order they were encountered, and construct +// transformers. +// o Some jargon from programming languages we can use here for illustration +// is that we are "hoisting" the main-flags as if they had been written on +// the command line before the verbs. +// +// We need to require a '--' between a verb and a main-flag so the main-flag +// doesn't look like a verb flag. For example, in 'mlr head -n 10 --csv +// foo.csv' the '--csv' looks like it belongs to the 'head' verb. When people +// use '#!/bin/sh' scripts they need to insert the '--' in 'mlr head -n 10 -- +// --csv foo.csv'; for 'mlr -s' we insert the '--' for them. + package climain import ( @@ -14,55 +82,193 @@ import ( // ParseCommandLine is the entrypoint for handling the Miller command line: // flags, verbs and their flags, and input file name(s). -func ParseCommandLine(args []string) ( - options cli.TOptions, +func ParseCommandLine( + args []string, +) ( + options *cli.TOptions, recordTransformers []transformers.IRecordTransformer, err error, ) { - options = cli.DefaultOptions() - argc := len(args) + // mlr -s scriptfile {data-file names ...} means take the contents of + // scriptfile as if it were command-line items. + args = maybeInterpolateDashS(args) + + // Pass one as described at the top of this file. + flagSequences, verbSequences, dataFileNames := parseCommandLinePassOne(args) + + // Pass two as described at the top of this file. + return parseCommandLinePassTwo(flagSequences, verbSequences, dataFileNames) +} + +// parseCommandLinePassOne is as described at the top of this file. +func parseCommandLinePassOne( + args []string, +) ( + flagSequences [][]string, + verbSequences [][]string, + dataFileNames []string, +) { + flagSequences = make([][]string, 0) + verbSequences = make([][]string, 0) + dataFileNames = make([]string, 0) + + // All verbs after the first must be preceded with "then" + onFirst := true + + // Throwaway options as described above: passed into the flag-table parser + // but we'll use for-real-use options in pass two. + options := cli.DefaultOptions() + argi := 1 + argc := len(args) - // Try .mlrrc overrides (then command-line on top of that). - // A --norc flag (if provided) must come before all other options. - // Or, they can set the environment variable MLRRC="__none__". - if argc >= 2 && args[argi] == "--norc" { - argi++ - } else { - loadMlrrcOrDie(&options) - } + for argi < argc /* variable increment within loop body */ { + + // Old argi is at start of sequence; argi will be after. + oargi := argi + + if args[argi][0] == '-' { + + if args[argi] == "--cpuprofile" { + // Already handled in main(); ignore here, and don't send it to pass two. + cli.CheckArgCount(args, argi, argc, 1) + argi += 2 + } else if args[argi] == "--version" { + // Exiting flag: handle it immediately. + fmt.Printf("mlr %s\n", version.STRING) + os.Exit(0) + } else if args[argi] == "--bare-version" { + // Exiting flag: handle it immediately. + fmt.Printf("%s\n", version.STRING) + os.Exit(0) + } else if help.ParseTerminalUsage(args[argi]) { + // Exiting flag: handle it immediately. + // Most help is in the 'mlr help' auxent but there are a few + // shorthands like 'mlr -h' and 'mlr -F'. + os.Exit(0) + + } else if args[argi] == "--norc" { + flagSequences = append(flagSequences, args[oargi:argi]) + argi += 1 + + } else if cli.FLAG_TABLE.Parse(args, argc, &argi, options) { + flagSequences = append(flagSequences, args[oargi:argi]) + + } else if args[argi] == "--" { + // This separates a main-flag from the verb/verb-flags before it + argi += 1 + + } else { + // Unrecognized main-flag. Fatal it here, and don't send it to pass two. + fmt.Fprintf(os.Stderr, "%s: option \"%s\" not recognized.\n", "mlr", args[argi]) + fmt.Fprintf(os.Stderr, "Please run \"%s --help\" for usage information.\n", "mlr") + os.Exit(1) + } - for argi < argc /* variable increment: 1 or 2 depending on flag */ { - if args[argi][0] != '-' { - break // No more flag options to process - - } else if args[argi] == "--cpuprofile" { - // Already handled in main(); ignore here. - cli.CheckArgCount(args, argi, argc, 1) - argi += 2 - } else if args[argi] == "--version" { - fmt.Printf("mlr %s\n", version.STRING) - os.Exit(0) - } else if args[argi] == "--bare-version" { - fmt.Printf("%s\n", version.STRING) - os.Exit(0) - - } else if help.ParseTerminalUsage(args[argi]) { - // Most help is in the 'mlr help' auxent but there are a few shorthands - // like 'mlr -h' and 'mlr -F'. - os.Exit(0) - - } else if cli.FLAG_TABLE.Parse(args, argc, &argi, &options) { - // handled + } else if onFirst || args[argi] == "then" { + // The first verb in the then-chain can *optionally* be preceded by + // 'then'. The others one *must* be. + if args[argi] == "then" { + cli.CheckArgCount(args, argi, argc, 1) + oargi++ + argi++ + } + verb := args[argi] + onFirst = false + + transformerSetup := transformers.LookUp(verb) + if transformerSetup == nil { + fmt.Fprintf(os.Stderr, + "%s: verb \"%s\" not found. Please use \"%s --help\" for a list.\n", + "mlr", verb, "mlr") + os.Exit(1) + } + + // It's up to the parse func to print its usage, and exit 1, on + // CLI-parse failure. Also note: this assumes main reader/writer opts + // are all parsed *before* transformer parse-CLI methods are invoked. + transformer := transformerSetup.ParseCLIFunc( + &argi, + argc, + args, + options, + false, // false for first pass of CLI-parse, true for second pass -- this is the first pass + ) + // For pass one we want the verbs to identify the arg-sequences + // they own within the command line, but not construct + // transformers. + lib.InternalCodingErrorIf(transformer != nil) + + verbSequences = append(verbSequences, args[oargi:argi]) } else { - // unhandled - fmt.Fprintf(os.Stderr, "%s: option \"%s\" not recognized.\n", "mlr", args[argi]) - fmt.Fprintf(os.Stderr, "Please run \"%s --help\" for usage information.\n", "mlr") - os.Exit(1) + // After main-flag sequences and verb sequences, data-file names + // still come last on the command line. + break } } + for ; argi < argc; argi++ { + dataFileNames = append(dataFileNames, args[argi]) + } + + if len(verbSequences) == 0 { + fmt.Fprintf(os.Stderr, "%s: no verb supplied.\n", "mlr") + help.MainUsage(os.Stderr) + os.Exit(1) + } + + return flagSequences, verbSequences, dataFileNames +} + +// parseCommandLinePassTwo is as described at the top of this file. +func parseCommandLinePassTwo( + flagSequences [][]string, + verbSequences [][]string, + dataFileNames []string, +) ( + options *cli.TOptions, + recordTransformers []transformers.IRecordTransformer, + err error, +) { + // Options take in-code defaults, then overridden by .mlrrc (if any and if + // desired), then those in turn overridden by command-line flags. + options = cli.DefaultOptions() + recordTransformers = make([]transformers.IRecordTransformer, 0) + err = nil + ignoresInput := false + + // Load a .mlrrc file unless --norc was a main-flag on the command line. + loadMlrrc := true + for _, flagSequence := range flagSequences { + lib.InternalCodingErrorIf(len(flagSequence) < 1) + if flagSequence[0] == "--norc" { + loadMlrrc = false + break + } + } + if loadMlrrc { + loadMlrrcOrDie(options) + } + + // Process the flag-sequences in order from pass one. We assume all the + // exiting flags like --help and --version were already processed, so all + // main-flags making it here to pass two are for the flag-table parser. + for _, flagSequence := range flagSequences { + argi := 0 + args := flagSequence + argc := len(args) + lib.InternalCodingErrorIf(argc == 0) + + // Parse the main-flag into the options struct. + rc := cli.FLAG_TABLE.Parse(args, argc, &argi, options) + + // Should have been parsed OK in pass one. + lib.InternalCodingErrorIf(rc != true) + // Make sure we consumed the entire flag sequence as parsed by pass one. + lib.InternalCodingErrorIf(argi != argc) + } + // Check now to avoid confusing timezone-library behavior later on lib.SetTZFromEnv() @@ -77,10 +283,40 @@ func ParseCommandLine(args []string) ( } } - recordTransformers, ignoresInput, err := parseTransformers(args, &argi, argc, &options) - if err != nil { - return options, recordTransformers, err + // Now process the verb-sequences from pass one, with options-struct set up + // and finalized. + for i, verbSequence := range verbSequences { + argi := 0 // xxx needed? + args := verbSequence + argc := len(args) + lib.InternalCodingErrorIf(argc == 0) + + // Non-existent verbs should have been fatalled in pass one. + transformerSetup := transformers.LookUp(args[0]) + lib.InternalCodingErrorIf(transformerSetup == nil) + + // It's up to the parse func to print its usage, and exit 1, on + // CLI-parse failure. + transformer := transformerSetup.ParseCLIFunc( + &argi, + argc, + args, + options, + true, // false for first pass of CLI-parse, true for second pass -- this is the first pass + ) + // Unparseable verb-setups should have been found in pass one. + lib.InternalCodingErrorIf(transformer == nil) + // Make sure we consumed the entire verb sequence as parsed by pass one. + lib.InternalCodingErrorIf(argi != argc) + + // E.g. then-chain begins with seqgen + if i == 0 && transformerSetup.IgnoresInput { + ignoresInput = true + } + + recordTransformers = append(recordTransformers, transformer) } + if ignoresInput { options.NoInput = true // e.g. then-chain begins with seqgen } @@ -94,7 +330,7 @@ func ParseCommandLine(args []string) ( recordTransformers = append(recordTransformers, transformer) } - if cli.DecideFinalUnflatten(&options) { + if cli.DecideFinalUnflatten(options) { // E.g. req.method=GET,req.path=/api/check becomes // '{"req": {"method": "GET", "path": "/api/check"}}' transformer, err := transformers.NewTransformerUnflatten(options.WriterOptions.FLATSEP, nil) @@ -105,8 +341,8 @@ func ParseCommandLine(args []string) ( // There may already be one or more because of --from on the command line, // so append. - for _, arg := range args[argi:] { - options.FileNames = append(options.FileNames, arg) + for _, dataFileName := range dataFileNames { + options.FileNames = append(options.FileNames, dataFileName) } // E.g. mlr -n put -v '...' @@ -125,89 +361,3 @@ func ParseCommandLine(args []string) ( return options, recordTransformers, nil } - -// parseTransformers returns a list of transformers, from the starting point in -// args given by *pargi. Bumps *pargi to point to remaining -// post-transformer-setup args, i.e. filenames. -func parseTransformers( - args []string, - pargi *int, - argc int, - options *cli.TOptions, -) ( - transformerList []transformers.IRecordTransformer, - ignoresInput bool, - err error, -) { - - transformerList = make([]transformers.IRecordTransformer, 0) - ignoresInput = false - - argi := *pargi - - // Allow then-chains to start with an initial 'then': 'mlr verb1 then verb2 then verb3' or - // 'mlr then verb1 then verb2 then verb3'. Particuarly useful in backslashy scripting contexts. - if (argc-argi) >= 1 && args[argi] == "then" { - argi++ - } - - if (argc - argi) < 1 { - fmt.Fprintf(os.Stderr, "%s: no verb supplied.\n", "mlr") - help.MainUsage(os.Stderr) - os.Exit(1) - } - - onFirst := true - - for { - cli.CheckArgCount(args, argi, argc, 1) - verb := args[argi] - - transformerSetup := transformers.LookUp(verb) - if transformerSetup == nil { - fmt.Fprintf(os.Stderr, - "%s: verb \"%s\" not found. Please use \"%s --help\" for a list.\n", - "mlr", verb, "mlr") - os.Exit(1) - } - - // E.g. then-chain begins with seqgen - if onFirst && transformerSetup.IgnoresInput { - ignoresInput = true - } - onFirst = false - - // It's up to the parse func to print its usage, and exit 1, on - // CLI-parse failure. Also note: this assumes main reader/writer opts - // are all parsed *before* transformer parse-CLI methods are invoked. - transformer := transformerSetup.ParseCLIFunc( - &argi, - argc, - args, - options, - ) - lib.InternalCodingErrorIf(transformer == nil) - - transformerList = append(transformerList, transformer) - - // argi now points to: - // * A "then", followed by the start of the next verb in the chain, if any - // * Filenames after the verb (there are no more verbs listed) - // * None of the above; argi == argc - if argi >= argc { - break - } else if args[argi] != "then" { - break - } else { - if argi == argc-1 { - fmt.Fprintf(os.Stderr, "%s: missing next verb after \"then\".\n", "mlr") - os.Exit(1) - } else { - argi++ - } - } - } - - *pargi = argi - return transformerList, ignoresInput, nil -} diff --git a/internal/pkg/climain/mlrcli_shebang.go b/internal/pkg/climain/mlrcli_shebang.go new file mode 100644 index 0000000000..88782be157 --- /dev/null +++ b/internal/pkg/climain/mlrcli_shebang.go @@ -0,0 +1,87 @@ +package climain + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/johnkerl/miller/internal/pkg/lib" + shellquote "github.com/kballard/go-shellquote" +) + +// maybeInterpolateDashS supports Miller scripts with shebang lines like +// #!/usr/bin/env mlr -s +// --csv tac then filter ' +// NR % 2 == 1 +// ' +// invoked as +// scriptfile input1.csv input2.csv +// The "-s" flag must be the very first command-line argument after "mlr" for +// two reasons: +// * This is how shebang lines work +// * There are Miller verbs with -s flags and we don't want to disrupt their behavior. +func maybeInterpolateDashS(args []string) []string { + if len(args) < 2 { + return args + } + if args[1] != "-s" { // Normal case + return args + } + if len(args) < 3 { + fmt.Fprintf(os.Stderr, "mlr: -s flag requires a filename after it.\n") + os.Exit(1) + } + + // mlr -s scriptfile input1.csv input2.csv + // 0 1 2 3 4 + arg0 := args[0] + filename := args[2] + remainingArgs := args[3:] + + // Read the bytes in the filename given after -s. + byteContents, rerr := ioutil.ReadFile(filename) + if rerr != nil { + fmt.Fprintf(os.Stderr, "mlr: cannot read %s: %v\n", filename, rerr) + os.Exit(1) + } + contents := string(byteContents) + + // Split into lines + contents = strings.ReplaceAll(contents, "\r\n", "\n") + lines := lib.SplitString(contents, "\n") + + // Remove the shebang line itself. + if len(lines) >= 1 { + if strings.HasPrefix(lines[0], "#!") { + lines = lines[1:] + } + } + + // TODO: maybe support comment lines deeper within the script-file. + // Make sure they're /^[\s]+#/ since we dont' want to disrupt a "#" within + // strings which are not actually comment characters. + + // Re-join lines to strings, and pass off to a shell-parser to split into + // an args[]-style array. + contents = strings.Join(lines, "\n") + argsFromFile, err := shellquote.Split(contents) + if err != nil { + fmt.Fprintf(os.Stderr, "mlr: cannot parse %s: %v\n", filename, err) + os.Exit(1) + } + + // Join "mlr", the args from the script-file contents, and all the remaining arguments + // on the original command line after "mlr -s {scriptfile}" + newArgs := make([]string, 1) + newArgs[0] = arg0 + + for _, argFromFile := range argsFromFile { + newArgs = append(newArgs, argFromFile) + } + for _, remainingArg := range remainingArgs { + newArgs = append(newArgs, remainingArg) + } + + return newArgs +} diff --git a/internal/pkg/entrypoint/entrypoint.go b/internal/pkg/entrypoint/entrypoint.go index ce0b56ce7d..26fb345658 100644 --- a/internal/pkg/entrypoint/entrypoint.go +++ b/internal/pkg/entrypoint/entrypoint.go @@ -61,10 +61,10 @@ func Main() { // processToStdout is normal processing without mlr -I. func processToStdout( - options cli.TOptions, + options *cli.TOptions, recordTransformers []transformers.IRecordTransformer, ) { - err := stream.Stream(options.FileNames, &options, recordTransformers, os.Stdout, true) + err := stream.Stream(options.FileNames, options, recordTransformers, os.Stdout, true) if err != nil { fmt.Fprintf(os.Stderr, "mlr: %v.\n", err) os.Exit(1) @@ -85,7 +85,7 @@ func processToStdout( // approach leads to greater code stability. func processInPlace( - originalOptions cli.TOptions, + originalOptions *cli.TOptions, ) { // This should have been already checked by the CLI parser when validating // the -I flag. @@ -148,7 +148,7 @@ func processInPlace( } // Run the Miller processing stream from the input file to the temp-output file. - err = stream.Stream([]string{fileName}, &options, recordTransformers, wrappedHandle, false) + err = stream.Stream([]string{fileName}, options, recordTransformers, wrappedHandle, false) if err != nil { os.Remove(tempFileName) fmt.Fprintf(os.Stderr, "mlr: %v\n", err) diff --git a/internal/pkg/transformers/aaa_record_transformer.go b/internal/pkg/transformers/aaa_record_transformer.go index 0a8d8cc844..7937f8c911 100644 --- a/internal/pkg/transformers/aaa_record_transformer.go +++ b/internal/pkg/transformers/aaa_record_transformer.go @@ -37,6 +37,7 @@ type TransformerParseCLIFunc func( argc int, args []string, mainOptions *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer type TransformerSetup struct { diff --git a/internal/pkg/transformers/altkv.go b/internal/pkg/transformers/altkv.go index e6ffbf63b2..0c9344f884 100644 --- a/internal/pkg/transformers/altkv.go +++ b/internal/pkg/transformers/altkv.go @@ -39,6 +39,7 @@ func transformerAltkvParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -50,6 +51,9 @@ func transformerAltkvParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -60,13 +64,17 @@ func transformerAltkvParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerAltkv() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/bar.go b/internal/pkg/transformers/bar.go index 9c140aaacf..70f3da2dfa 100644 --- a/internal/pkg/transformers/bar.go +++ b/internal/pkg/transformers/bar.go @@ -60,6 +60,7 @@ func transformerBarParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -82,6 +83,9 @@ func transformerBarParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -116,6 +120,11 @@ func transformerBarParseCLI( transformerBarUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerBar( fieldNames, lo, @@ -131,7 +140,6 @@ func transformerBarParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/bootstrap.go b/internal/pkg/transformers/bootstrap.go index 975b9b7729..b379bb0fef 100644 --- a/internal/pkg/transformers/bootstrap.go +++ b/internal/pkg/transformers/bootstrap.go @@ -48,6 +48,7 @@ func transformerBootstrapParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -62,6 +63,9 @@ func transformerBootstrapParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -75,13 +79,17 @@ func transformerBootstrapParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerBootstrap(nout) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/cat.go b/internal/pkg/transformers/cat.go index 21930207d7..f16aabfb8a 100644 --- a/internal/pkg/transformers/cat.go +++ b/internal/pkg/transformers/cat.go @@ -42,6 +42,7 @@ func transformerCatParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -59,6 +60,9 @@ func transformerCatParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -78,6 +82,11 @@ func transformerCatParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerCat( doCounters, counterFieldName, @@ -88,7 +97,6 @@ func transformerCatParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/check.go b/internal/pkg/transformers/check.go index 983619f999..7c2238f417 100644 --- a/internal/pkg/transformers/check.go +++ b/internal/pkg/transformers/check.go @@ -40,6 +40,7 @@ func transformerCheckParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -51,6 +52,9 @@ func transformerCheckParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -61,13 +65,17 @@ func transformerCheckParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerCheck() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/clean-whitespace.go b/internal/pkg/transformers/clean-whitespace.go index fb182adca5..3df93ab4c2 100644 --- a/internal/pkg/transformers/clean-whitespace.go +++ b/internal/pkg/transformers/clean-whitespace.go @@ -48,6 +48,7 @@ func transformerCleanWhitespaceParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { doKeys := true @@ -62,6 +63,9 @@ func transformerCleanWhitespaceParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -83,6 +87,11 @@ func transformerCleanWhitespaceParseCLI( transformerCleanWhitespaceUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerCleanWhitespace( doKeys, doValues, @@ -92,7 +101,6 @@ func transformerCleanWhitespaceParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/count-similar.go b/internal/pkg/transformers/count-similar.go index a78d5e8d5b..2d7cf4ebf0 100644 --- a/internal/pkg/transformers/count-similar.go +++ b/internal/pkg/transformers/count-similar.go @@ -44,6 +44,7 @@ func transformerCountSimilarParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -59,6 +60,9 @@ func transformerCountSimilarParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -79,6 +83,11 @@ func transformerCountSimilarParseCLI( transformerCountSimilarUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerCountSimilar( groupByFieldNames, counterFieldName, @@ -88,7 +97,6 @@ func transformerCountSimilarParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/count.go b/internal/pkg/transformers/count.go index 85d373e437..1134cd4157 100644 --- a/internal/pkg/transformers/count.go +++ b/internal/pkg/transformers/count.go @@ -45,6 +45,7 @@ func transformerCountParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -61,6 +62,9 @@ func transformerCountParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -80,6 +84,11 @@ func transformerCountParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerCount( groupByFieldNames, showCountsOnly, @@ -90,7 +99,6 @@ func transformerCountParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/cut.go b/internal/pkg/transformers/cut.go index 2c2ecc8102..7aa5a833aa 100644 --- a/internal/pkg/transformers/cut.go +++ b/internal/pkg/transformers/cut.go @@ -55,6 +55,7 @@ func transformerCutParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -72,6 +73,9 @@ func transformerCutParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -101,6 +105,11 @@ func transformerCutParseCLI( transformerCutUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerCut( fieldNames, doArgOrder, @@ -112,7 +121,6 @@ func transformerCutParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/decimate.go b/internal/pkg/transformers/decimate.go index fdd228ae29..051c6fdbb6 100644 --- a/internal/pkg/transformers/decimate.go +++ b/internal/pkg/transformers/decimate.go @@ -43,6 +43,7 @@ func transformerDecimateParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -60,6 +61,9 @@ func transformerDecimateParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -85,6 +89,11 @@ func transformerDecimateParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerDecimate( decimateCount, atStart, @@ -96,7 +105,6 @@ func transformerDecimateParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/fill-down.go b/internal/pkg/transformers/fill-down.go index 77b91b13a4..f21f516104 100644 --- a/internal/pkg/transformers/fill-down.go +++ b/internal/pkg/transformers/fill-down.go @@ -49,6 +49,7 @@ func transformerFillDownParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -65,6 +66,9 @@ func transformerFillDownParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -91,6 +95,11 @@ func transformerFillDownParseCLI( transformerFillDownUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerFillDown( fillDownFieldNames, doAll, @@ -101,7 +110,6 @@ func transformerFillDownParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/fill-empty.go b/internal/pkg/transformers/fill-empty.go index a2f79817de..e947d978a6 100644 --- a/internal/pkg/transformers/fill-empty.go +++ b/internal/pkg/transformers/fill-empty.go @@ -41,6 +41,7 @@ func transformerFillEmptyParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -56,6 +57,9 @@ func transformerFillEmptyParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -72,13 +76,17 @@ func transformerFillEmptyParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerFillEmpty(fillString, inferType) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/flatten.go b/internal/pkg/transformers/flatten.go index 0d9ef3693b..aeec5656f6 100644 --- a/internal/pkg/transformers/flatten.go +++ b/internal/pkg/transformers/flatten.go @@ -45,6 +45,7 @@ func transformerFlattenParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -60,6 +61,9 @@ func transformerFlattenParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -76,6 +80,11 @@ func transformerFlattenParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerFlatten( oFlatSep, fieldNames, @@ -85,7 +94,6 @@ func transformerFlattenParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/format-values.go b/internal/pkg/transformers/format-values.go index 1630ae6912..4aed82aee1 100644 --- a/internal/pkg/transformers/format-values.go +++ b/internal/pkg/transformers/format-values.go @@ -69,6 +69,7 @@ func transformerFormatValuesParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -86,6 +87,9 @@ func transformerFormatValuesParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -105,6 +109,11 @@ func transformerFormatValuesParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerFormatValues( stringFormat, intFormat, @@ -116,7 +125,6 @@ func transformerFormatValuesParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/fraction.go b/internal/pkg/transformers/fraction.go index 3a9c54a314..9534898780 100644 --- a/internal/pkg/transformers/fraction.go +++ b/internal/pkg/transformers/fraction.go @@ -59,6 +59,7 @@ func transformerFractionParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -77,6 +78,9 @@ func transformerFractionParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -103,6 +107,11 @@ func transformerFractionParseCLI( transformerFractionUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerFraction( fractionFieldNames, groupByFieldNames, @@ -114,7 +123,6 @@ func transformerFractionParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/gap.go b/internal/pkg/transformers/gap.go index 98e40e619c..34c6782eb0 100644 --- a/internal/pkg/transformers/gap.go +++ b/internal/pkg/transformers/gap.go @@ -45,6 +45,7 @@ func transformerGapParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -60,6 +61,9 @@ func transformerGapParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -80,6 +84,11 @@ func transformerGapParseCLI( transformerGapUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerGap( gapCount, groupByFieldNames, @@ -89,7 +98,6 @@ func transformerGapParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/grep.go b/internal/pkg/transformers/grep.go index 8455e403a9..2032e8bfa5 100644 --- a/internal/pkg/transformers/grep.go +++ b/internal/pkg/transformers/grep.go @@ -56,6 +56,7 @@ func transformerGrepParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -71,6 +72,9 @@ func transformerGrepParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -106,6 +110,11 @@ func transformerGrepParseCLI( os.Exit(1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerGrep( regexp, invert, @@ -115,7 +124,6 @@ func transformerGrepParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/group-by.go b/internal/pkg/transformers/group-by.go index c1dddf685c..3f79f92199 100644 --- a/internal/pkg/transformers/group-by.go +++ b/internal/pkg/transformers/group-by.go @@ -41,6 +41,7 @@ func transformerGroupByParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -52,6 +53,9 @@ func transformerGroupByParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -69,6 +73,11 @@ func transformerGroupByParseCLI( groupByFieldNames := lib.SplitString(args[argi], ",") argi++ + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerGroupBy( groupByFieldNames, ) @@ -77,7 +86,6 @@ func transformerGroupByParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/group-like.go b/internal/pkg/transformers/group-like.go index 3b03c078f0..6946a99620 100644 --- a/internal/pkg/transformers/group-like.go +++ b/internal/pkg/transformers/group-like.go @@ -41,6 +41,7 @@ func transformerGroupLikeParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -52,6 +53,9 @@ func transformerGroupLikeParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -62,13 +66,17 @@ func transformerGroupLikeParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerGroupLike() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/having-fields.go b/internal/pkg/transformers/having-fields.go index 26d28a8ce5..cdd8986903 100644 --- a/internal/pkg/transformers/having-fields.go +++ b/internal/pkg/transformers/having-fields.go @@ -67,6 +67,7 @@ func transformerHavingFieldsParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { havingFieldsCriterion := havingFieldsCriterionUnspecified @@ -83,6 +84,9 @@ func transformerHavingFieldsParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -130,6 +134,11 @@ func transformerHavingFieldsParseCLI( transformerHavingFieldsUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerHavingFields( havingFieldsCriterion, fieldNames, @@ -140,7 +149,6 @@ func transformerHavingFieldsParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/head.go b/internal/pkg/transformers/head.go index 25c953bcd5..14b3713fa0 100644 --- a/internal/pkg/transformers/head.go +++ b/internal/pkg/transformers/head.go @@ -43,6 +43,7 @@ func transformerHeadParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -58,6 +59,9 @@ func transformerHeadParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -74,6 +78,11 @@ func transformerHeadParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerHead( headCount, groupByFieldNames, @@ -83,7 +92,6 @@ func transformerHeadParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/histogram.go b/internal/pkg/transformers/histogram.go index 7bd6a1e0bc..6262a6975e 100644 --- a/internal/pkg/transformers/histogram.go +++ b/internal/pkg/transformers/histogram.go @@ -49,6 +49,7 @@ func transformerHistogramParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -69,6 +70,9 @@ func transformerHistogramParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -109,6 +113,11 @@ func transformerHistogramParseCLI( transformerHistogramUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerHistogram( valueFieldNames, lo, @@ -122,7 +131,6 @@ func transformerHistogramParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/join.go b/internal/pkg/transformers/join.go index 4a067dc8c8..243bbd7443 100644 --- a/internal/pkg/transformers/join.go +++ b/internal/pkg/transformers/join.go @@ -137,6 +137,7 @@ func transformerJoinParseCLI( argc int, args []string, mainOptions *cli.TOptions, // Options for the right-files + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -157,6 +158,9 @@ func transformerJoinParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -256,13 +260,17 @@ func transformerJoinParseCLI( os.Exit(1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerJoin(opts) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/json-parse.go b/internal/pkg/transformers/json-parse.go index 4b3e81b853..fa8c52415d 100644 --- a/internal/pkg/transformers/json-parse.go +++ b/internal/pkg/transformers/json-parse.go @@ -44,6 +44,7 @@ func transformerJSONParseParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -58,6 +59,9 @@ func transformerJSONParseParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -71,6 +75,11 @@ func transformerJSONParseParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerJSONParse( fieldNames, ) @@ -79,7 +88,6 @@ func transformerJSONParseParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/json-stringify.go b/internal/pkg/transformers/json-stringify.go index 17e80247e7..ada2a1ff58 100644 --- a/internal/pkg/transformers/json-stringify.go +++ b/internal/pkg/transformers/json-stringify.go @@ -45,6 +45,7 @@ func transformerJSONStringifyParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -60,6 +61,9 @@ func transformerJSONStringifyParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -86,6 +90,11 @@ func transformerJSONStringifyParseCLI( jsonFormatting = types.JSON_SINGLE_LINE } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerJSONStringify( jsonFormatting, fieldNames, @@ -95,7 +104,6 @@ func transformerJSONStringifyParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/label.go b/internal/pkg/transformers/label.go index 5348fc8640..39c70e550b 100644 --- a/internal/pkg/transformers/label.go +++ b/internal/pkg/transformers/label.go @@ -45,6 +45,7 @@ func transformerLabelParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -56,6 +57,9 @@ func transformerLabelParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -73,6 +77,11 @@ func transformerLabelParseCLI( newNames := lib.SplitString(args[argi], ",") argi++ + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerLabel( newNames, ) @@ -82,7 +91,6 @@ func transformerLabelParseCLI( // TODO: return nil to caller and have it exit, maybe } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/merge-fields.go b/internal/pkg/transformers/merge-fields.go index 7cc09ae02e..56f2846a32 100644 --- a/internal/pkg/transformers/merge-fields.go +++ b/internal/pkg/transformers/merge-fields.go @@ -82,6 +82,7 @@ func transformerMergeFieldsParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -101,6 +102,9 @@ func transformerMergeFieldsParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -158,6 +162,11 @@ func transformerMergeFieldsParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerMergeFields( accumulatorNameList, valueFieldNameList, @@ -171,7 +180,6 @@ func transformerMergeFieldsParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/most-or-least-frequent.go b/internal/pkg/transformers/most-or-least-frequent.go index dd35a847f7..80929ba4c7 100644 --- a/internal/pkg/transformers/most-or-least-frequent.go +++ b/internal/pkg/transformers/most-or-least-frequent.go @@ -79,8 +79,9 @@ func transformerMostFrequentParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { - return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, true, transformerMostFrequentUsage) + return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, true, transformerMostFrequentUsage, doConstruct) } func transformerLeastFrequentParseCLI( @@ -88,8 +89,9 @@ func transformerLeastFrequentParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { - return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, false, transformerLeastFrequentUsage) + return transformerMostOrLeastFrequentParseCLI(pargi, argc, args, false, transformerLeastFrequentUsage, doConstruct) } func transformerMostOrLeastFrequentParseCLI( @@ -98,6 +100,7 @@ func transformerMostOrLeastFrequentParseCLI( args []string, descending bool, usageFunc TransformerUsageFunc, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -116,6 +119,9 @@ func transformerMostOrLeastFrequentParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -143,6 +149,11 @@ func transformerMostOrLeastFrequentParseCLI( return nil } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerMostOrLeastFrequent( groupByFieldNames, maxOutputLength, @@ -155,7 +166,6 @@ func transformerMostOrLeastFrequentParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/nest.go b/internal/pkg/transformers/nest.go index 72e0d28649..35c427ebbd 100644 --- a/internal/pkg/transformers/nest.go +++ b/internal/pkg/transformers/nest.go @@ -97,6 +97,7 @@ func transformerNestParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -122,6 +123,9 @@ func transformerNestParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -196,6 +200,11 @@ func transformerNestParseCLI( transformerNestUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerNest( fieldName, nestedFS, @@ -209,7 +218,6 @@ func transformerNestParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/nothing.go b/internal/pkg/transformers/nothing.go index 547eee0a9c..c3cead81f9 100644 --- a/internal/pkg/transformers/nothing.go +++ b/internal/pkg/transformers/nothing.go @@ -40,6 +40,7 @@ func transformerNothingParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -51,6 +52,9 @@ func transformerNothingParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -61,13 +65,17 @@ func transformerNothingParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerNothing() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/put-or-filter.go b/internal/pkg/transformers/put-or-filter.go index 4ffd14f369..1702d9ccaf 100644 --- a/internal/pkg/transformers/put-or-filter.go +++ b/internal/pkg/transformers/put-or-filter.go @@ -190,6 +190,7 @@ func transformerPutOrFilterParseCLI( argc int, args []string, mainOptions *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -236,6 +237,9 @@ func transformerPutOrFilterParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -323,13 +327,19 @@ func transformerPutOrFilterParseCLI( if !haveDSLStringsHere { // Get the DSL string from the command line, after the flags if argi >= argc { - transformerPutUsage(os.Stderr, true, 1) + fmt.Fprintf(os.Stderr, "mlr %s: -f/-e requires a filename as argument.\n", verb) + os.Exit(1) } dslString := args[argi] dslStrings = append(dslStrings, dslString) argi++ } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + var dslInstanceType cst.DSLInstanceType = cst.DSLInstanceTypePut if verb == "filter" { dslInstanceType = cst.DSLInstanceTypeFilter @@ -355,7 +365,6 @@ func transformerPutOrFilterParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/regularize.go b/internal/pkg/transformers/regularize.go index 9e120d6875..b0ccf4c854 100644 --- a/internal/pkg/transformers/regularize.go +++ b/internal/pkg/transformers/regularize.go @@ -40,6 +40,7 @@ func transformerRegularizeParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -51,6 +52,9 @@ func transformerRegularizeParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -61,13 +65,17 @@ func transformerRegularizeParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerRegularize() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/remove-empty-columns.go b/internal/pkg/transformers/remove-empty-columns.go index 2b33125e7a..076bcbabb8 100644 --- a/internal/pkg/transformers/remove-empty-columns.go +++ b/internal/pkg/transformers/remove-empty-columns.go @@ -40,6 +40,7 @@ func transformerRemoveEmptyColumnsParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -51,6 +52,9 @@ func transformerRemoveEmptyColumnsParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -61,13 +65,17 @@ func transformerRemoveEmptyColumnsParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerRemoveEmptyColumns() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/rename.go b/internal/pkg/transformers/rename.go index c2e90019bd..ab860fc391 100644 --- a/internal/pkg/transformers/rename.go +++ b/internal/pkg/transformers/rename.go @@ -63,6 +63,7 @@ func transformerRenameParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -77,6 +78,9 @@ func transformerRenameParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -107,6 +111,11 @@ func transformerRenameParseCLI( } argi++ + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerRename( names, doRegexes, @@ -117,7 +126,6 @@ func transformerRenameParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/reorder.go b/internal/pkg/transformers/reorder.go index 91add0190d..680849b46e 100644 --- a/internal/pkg/transformers/reorder.go +++ b/internal/pkg/transformers/reorder.go @@ -56,6 +56,7 @@ func transformerReorderParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -73,6 +74,9 @@ func transformerReorderParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -105,6 +109,11 @@ func transformerReorderParseCLI( transformerReorderUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerReorder( fieldNames, putAtEnd, @@ -116,7 +125,6 @@ func transformerReorderParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/repeat.go b/internal/pkg/transformers/repeat.go index 4f51df0a19..7df62d54a4 100644 --- a/internal/pkg/transformers/repeat.go +++ b/internal/pkg/transformers/repeat.go @@ -69,6 +69,7 @@ func transformerRepeatParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { repeatCountSource := repeatCountSourceUnspecified @@ -85,6 +86,9 @@ func transformerRepeatParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -107,6 +111,11 @@ func transformerRepeatParseCLI( transformerRepeatUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerRepeat( repeatCountSource, repeatCount, @@ -117,7 +126,6 @@ func transformerRepeatParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/reshape.go b/internal/pkg/transformers/reshape.go index a403b4ba21..d78861de14 100644 --- a/internal/pkg/transformers/reshape.go +++ b/internal/pkg/transformers/reshape.go @@ -123,6 +123,7 @@ func transformerReshapeParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -141,6 +142,9 @@ func transformerReshapeParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -188,6 +192,11 @@ func transformerReshapeParseCLI( splitOutValueFieldName = splitOutFieldNames[1] } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerReshape( inputFieldNames, inputFieldRegexStrings, @@ -201,7 +210,6 @@ func transformerReshapeParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/sample.go b/internal/pkg/transformers/sample.go index 85e7b73adc..8efbc45c66 100644 --- a/internal/pkg/transformers/sample.go +++ b/internal/pkg/transformers/sample.go @@ -45,6 +45,7 @@ func transformerSampleParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -60,6 +61,9 @@ func transformerSampleParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -80,6 +84,11 @@ func transformerSampleParseCLI( transformerSampleUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSample( sampleCount, groupByFieldNames, @@ -89,7 +98,6 @@ func transformerSampleParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/sec2gmt.go b/internal/pkg/transformers/sec2gmt.go index 9f02b59539..8965d7397f 100644 --- a/internal/pkg/transformers/sec2gmt.go +++ b/internal/pkg/transformers/sec2gmt.go @@ -48,6 +48,7 @@ func transformerSec2GMTParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -62,6 +63,9 @@ func transformerSec2GMTParseCLI( if opt[0] != '-' { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -104,6 +108,11 @@ func transformerSec2GMTParseCLI( fieldNames := args[argi] argi++ + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSec2GMT( fieldNames, preDivide, @@ -114,7 +123,6 @@ func transformerSec2GMTParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/sec2gmtdate.go b/internal/pkg/transformers/sec2gmtdate.go index 74be2d9443..d3a3f65f38 100644 --- a/internal/pkg/transformers/sec2gmtdate.go +++ b/internal/pkg/transformers/sec2gmtdate.go @@ -42,6 +42,7 @@ func transformerSec2GMTDateParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -53,6 +54,9 @@ func transformerSec2GMTDateParseCLI( if opt[0] != '-' { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -69,6 +73,11 @@ func transformerSec2GMTDateParseCLI( fieldNames := args[argi] argi++ + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSec2GMTDate( fieldNames, ) @@ -77,7 +86,6 @@ func transformerSec2GMTDateParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/seqgen.go b/internal/pkg/transformers/seqgen.go index aa3a898206..13227bd8e3 100644 --- a/internal/pkg/transformers/seqgen.go +++ b/internal/pkg/transformers/seqgen.go @@ -51,6 +51,7 @@ func transformerSeqgenParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -68,6 +69,9 @@ func transformerSeqgenParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -90,6 +94,11 @@ func transformerSeqgenParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSeqgen( fieldName, startString, @@ -101,7 +110,6 @@ func transformerSeqgenParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/shuffle.go b/internal/pkg/transformers/shuffle.go index 44f93fc964..261c172dbb 100644 --- a/internal/pkg/transformers/shuffle.go +++ b/internal/pkg/transformers/shuffle.go @@ -44,6 +44,7 @@ func transformerShuffleParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -55,6 +56,9 @@ func transformerShuffleParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -65,13 +69,17 @@ func transformerShuffleParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerShuffle() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/skip-trivial-records.go b/internal/pkg/transformers/skip-trivial-records.go index 8f5ab8360c..69d68c01d1 100644 --- a/internal/pkg/transformers/skip-trivial-records.go +++ b/internal/pkg/transformers/skip-trivial-records.go @@ -40,6 +40,7 @@ func transformerSkipTrivialRecordsParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -51,6 +52,9 @@ func transformerSkipTrivialRecordsParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -61,13 +65,17 @@ func transformerSkipTrivialRecordsParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSkipTrivialRecords() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/sort-within-records.go b/internal/pkg/transformers/sort-within-records.go index 14787cd0c4..de520085b8 100644 --- a/internal/pkg/transformers/sort-within-records.go +++ b/internal/pkg/transformers/sort-within-records.go @@ -40,6 +40,7 @@ func transformerSortWithinRecordsParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -52,6 +53,9 @@ func transformerSortWithinRecordsParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -68,13 +72,17 @@ func transformerSortWithinRecordsParseCLI( // TODO: allow sort by key or value? // TODO: allow sort ascendending/descending? + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSortWithinRecords(doRecurse) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/sort.go b/internal/pkg/transformers/sort.go index ac39ed29a0..0e9d54e2a8 100644 --- a/internal/pkg/transformers/sort.go +++ b/internal/pkg/transformers/sort.go @@ -100,6 +100,7 @@ func transformerSortParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -115,6 +116,9 @@ func transformerSortParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -227,6 +231,11 @@ func transformerSortParseCLI( transformerSortUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerSort( groupByFieldNames, comparatorFuncs, @@ -236,7 +245,6 @@ func transformerSortParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/stats1.go b/internal/pkg/transformers/stats1.go index 2ea2a33cca..07373a3d7d 100644 --- a/internal/pkg/transformers/stats1.go +++ b/internal/pkg/transformers/stats1.go @@ -97,6 +97,7 @@ func transformerStats1ParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -121,6 +122,9 @@ func transformerStats1ParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -187,6 +191,11 @@ func transformerStats1ParseCLI( os.Exit(1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerStats1( accumulatorNameList, valueFieldNameList, @@ -205,7 +214,6 @@ func transformerStats1ParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/stats2.go b/internal/pkg/transformers/stats2.go index b56ef5eeea..c4d63571a7 100644 --- a/internal/pkg/transformers/stats2.go +++ b/internal/pkg/transformers/stats2.go @@ -69,6 +69,7 @@ func transformerStats2ParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -90,6 +91,9 @@ func transformerStats2ParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -146,6 +150,11 @@ func transformerStats2ParseCLI( os.Exit(1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerStats2( accumulatorNameList, valueFieldNameList, @@ -159,7 +168,6 @@ func transformerStats2ParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/step.go b/internal/pkg/transformers/step.go index d1ce5fcc63..e2eeb60e6d 100644 --- a/internal/pkg/transformers/step.go +++ b/internal/pkg/transformers/step.go @@ -80,6 +80,7 @@ func transformerStepParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -98,6 +99,9 @@ func transformerStepParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -127,6 +131,11 @@ func transformerStepParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerStep( stepperNames, valueFieldNames, @@ -139,7 +148,6 @@ func transformerStepParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/tac.go b/internal/pkg/transformers/tac.go index 21861f6a9c..8f9395253a 100644 --- a/internal/pkg/transformers/tac.go +++ b/internal/pkg/transformers/tac.go @@ -40,6 +40,7 @@ func transformerTacParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -51,6 +52,9 @@ func transformerTacParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -61,13 +65,17 @@ func transformerTacParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerTac() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/tail.go b/internal/pkg/transformers/tail.go index 4ede585c14..4b369f7f38 100644 --- a/internal/pkg/transformers/tail.go +++ b/internal/pkg/transformers/tail.go @@ -44,6 +44,7 @@ func transformerTailParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -59,6 +60,9 @@ func transformerTailParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -75,6 +79,11 @@ func transformerTailParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerTail( tailCount, groupByFieldNames, @@ -84,7 +93,6 @@ func transformerTailParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/tee.go b/internal/pkg/transformers/tee.go index 0ee822bd48..f8e0b55ca1 100644 --- a/internal/pkg/transformers/tee.go +++ b/internal/pkg/transformers/tee.go @@ -47,6 +47,7 @@ func transformerTeeParseCLI( argc int, args []string, mainOptions *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -70,6 +71,9 @@ func transformerTeeParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -107,6 +111,11 @@ func transformerTeeParseCLI( filenameOrCommand = args[argi] argi++ + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerTee( appending, piping, @@ -118,7 +127,6 @@ func transformerTeeParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/template.go b/internal/pkg/transformers/template.go index 78d69d986b..2934d0427c 100644 --- a/internal/pkg/transformers/template.go +++ b/internal/pkg/transformers/template.go @@ -49,6 +49,7 @@ func transformerTemplateParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -64,6 +65,9 @@ func transformerTemplateParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -93,6 +97,11 @@ func transformerTemplateParseCLI( transformerTemplateUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerTemplate( fieldNames, fillWith, @@ -102,7 +111,6 @@ func transformerTemplateParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/top.go b/internal/pkg/transformers/top.go index 679b9ad0ad..bbd0ba37fa 100644 --- a/internal/pkg/transformers/top.go +++ b/internal/pkg/transformers/top.go @@ -53,6 +53,7 @@ func transformerTopParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -73,6 +74,9 @@ func transformerTopParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -107,6 +111,11 @@ func transformerTopParseCLI( transformerTopUsage(os.Stderr, true, 1) } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, _ := NewTransformerTop( topCount, valueFieldNames, @@ -116,7 +125,6 @@ func transformerTopParseCLI( outputFieldName, ) - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/unflatten.go b/internal/pkg/transformers/unflatten.go index 7de69552bb..67da3584bd 100644 --- a/internal/pkg/transformers/unflatten.go +++ b/internal/pkg/transformers/unflatten.go @@ -45,6 +45,7 @@ func transformerUnflattenParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -60,6 +61,9 @@ func transformerUnflattenParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -76,6 +80,11 @@ func transformerUnflattenParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerUnflatten( oFlatSep, fieldNames, @@ -85,7 +94,6 @@ func transformerUnflattenParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/uniq.go b/internal/pkg/transformers/uniq.go index 433552c6a5..e68a55b9cc 100644 --- a/internal/pkg/transformers/uniq.go +++ b/internal/pkg/transformers/uniq.go @@ -62,6 +62,7 @@ func transformerCountDistinctParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -80,6 +81,9 @@ func transformerCountDistinctParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -112,6 +116,11 @@ func transformerCountDistinctParseCLI( showCounts := true uniqifyEntireRecords := false + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerUniq( fieldNames, showCounts, @@ -125,7 +134,6 @@ func transformerCountDistinctParseCLI( os.Exit(1) } - *pargi = argi return transformer } @@ -161,6 +169,7 @@ func transformerUniqParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -180,6 +189,9 @@ func transformerUniqParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -220,6 +232,11 @@ func transformerUniqParseCLI( doLashed := true + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, _ := NewTransformerUniq( fieldNames, showCounts, @@ -229,7 +246,6 @@ func transformerUniqParseCLI( uniqifyEntireRecords, ) - *pargi = argi return transformer } diff --git a/internal/pkg/transformers/unsparsify.go b/internal/pkg/transformers/unsparsify.go index aee99b4252..a3f740908f 100644 --- a/internal/pkg/transformers/unsparsify.go +++ b/internal/pkg/transformers/unsparsify.go @@ -56,6 +56,7 @@ func transformerUnsparsifyParseCLI( argc int, args []string, _ *cli.TOptions, + doConstruct bool, // false for first pass of CLI-parse, true for second pass ) IRecordTransformer { // Skip the verb name from the current spot in the mlr command line @@ -71,6 +72,9 @@ func transformerUnsparsifyParseCLI( if !strings.HasPrefix(opt, "-") { break // No more flag options to process } + if args[argi] == "--" { + break // All transformers must do this so main-flags can follow verb-flags + } argi++ if opt == "-h" || opt == "--help" { @@ -87,6 +91,11 @@ func transformerUnsparsifyParseCLI( } } + *pargi = argi + if !doConstruct { // All transformers must do this for main command-line parsing + return nil + } + transformer, err := NewTransformerUnsparsify( fillerString, specifiedFieldNames, @@ -96,7 +105,6 @@ func transformerUnsparsifyParseCLI( os.Exit(1) } - *pargi = argi return transformer } diff --git a/man/manpage.txt b/man/manpage.txt index 4afa7a93a3..1e1addc4b0 100644 --- a/man/manpage.txt +++ b/man/manpage.txt @@ -2958,4 +2958,4 @@ SEE ALSO - 2021-11-23 MILLER(1) + 2021-11-24 MILLER(1) diff --git a/man/mlr.1 b/man/mlr.1 index 763e17f4bf..19ad3d220f 100644 --- a/man/mlr.1 +++ b/man/mlr.1 @@ -2,12 +2,12 @@ .\" Title: mlr .\" Author: [see the "AUTHOR" section] .\" Generator: ./mkman.rb -.\" Date: 2021-11-23 +.\" Date: 2021-11-24 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "MILLER" "1" "2021-11-23" "\ \&" "\ \&" +.TH "MILLER" "1" "2021-11-24" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Portability definitions .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/todo.txt b/todo.txt index f6253bd0c4..23d17904a1 100644 --- a/todo.txt +++ b/todo.txt @@ -12,6 +12,16 @@ PUNCHDOWN LIST - 0b1011 olh/webdoc - rtd links somewhere +* shebang + - maybe multiple -s -- ? []args would continue growing during parse ... + - try mlr -s on windows; maybe consolation for shebang-bash -- ? + - doc re where to put --norc + > it needn't be first (after -s) anymore + - olh + - webdocs + - want: + > myscript.mlr --c2j input.csv + * release ordering? conda brew macports chocolatey @@ -303,6 +313,12 @@ MAYBES ---------------------------------------------------------------- DEFER: + +* once on go 1.16: get around ioutil.ReadFile depcreation + o build-dsl hand-edit / sedder + o io/ioutil -> os + o ioutil.ReadFile -> os.ReadFile on internal/pkg/parsing/lexer/lexer.go + * parser-fu: o iterative LALR grok - jackson notes