diff --git a/go.mod b/go.mod index 55510973..cc9f0fee 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.24.0 require ( emperror.dev/errors v0.8.1 - github.com/Jeffail/gabs/v2 v2.7.0 github.com/NYTimes/logrotate v1.0.0 github.com/acobaugh/osrelease v0.1.0 github.com/apex/log v1.9.0 @@ -41,6 +40,8 @@ require ( github.com/shirou/gopsutil/v3 v3.24.5 github.com/spf13/cobra v1.10.1 github.com/stretchr/testify v1.11.1 + github.com/tidwall/gjson v1.18.0 + github.com/tidwall/sjson v1.2.5 golang.org/x/crypto v0.41.0 golang.org/x/sync v0.16.0 golang.org/x/sys v0.35.0 @@ -72,6 +73,8 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect ) diff --git a/go.sum b/go.sum index c8e0733c..a560d283 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= -github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= @@ -407,6 +405,15 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= diff --git a/parser/helpers.go b/parser/helpers.go index be09c686..a7758034 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -1,16 +1,16 @@ package parser import ( - "bytes" "regexp" "strconv" "strings" "emperror.dev/errors" - "github.com/Jeffail/gabs/v2" "github.com/apex/log" "github.com/buger/jsonparser" "github.com/iancoleman/strcase" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" ) // Regex to match anything that has a value matching the format of {{ config.$1 }} which @@ -62,12 +62,13 @@ func (cfr *ConfigurationFileReplacement) getKeyValue(value string) interface{} { // This does not currently support nested wildcard matches. For example, foo.*.bar // will work, however foo.*.bar.*.baz will not, since we'll only be splitting at the // first wildcard, and not subsequent ones. -func (f *ConfigurationFile) IterateOverJson(data []byte) (*gabs.Container, error) { - parsed, err := gabs.ParseJSON(data) - if err != nil { - return nil, err +func (f *ConfigurationFile) IterateOverJson(data []byte) ([]byte, error) { + if !gjson.ValidBytes(data) { + return nil, errors.New("invalid JSON data") } + jsonStr := string(data) + for _, v := range f.Replace { value, err := f.LookupConfigurationValue(v) if err != nil { @@ -78,140 +79,109 @@ func (f *ConfigurationFile) IterateOverJson(data []byte) (*gabs.Container, error // begin doing a search and replace in the data. if strings.Contains(v.Match, ".*") { parts := strings.SplitN(v.Match, ".*", 2) + basePath := strings.Trim(parts[0], ".") + remainingPath := strings.Trim(parts[1], ".") + + result := gjson.Get(jsonStr, basePath) + if !result.Exists() { + continue + } - // Iterate over each matched child and set the remaining path to the value - // that is passed through in the loop. - // - // If the child is a null value, nothing will happen. Seems reasonable as of the - // time this code is being written. - for _, child := range parsed.Path(strings.Trim(parts[0], ".")).Children() { - if err := v.SetAtPathway(child, strings.Trim(parts[1], "."), value); err != nil { - if errors.Is(err, gabs.ErrNotFound) { - continue + if result.IsArray() { + result.ForEach(func(key, val gjson.Result) bool { + fullPath := basePath + "." + key.String() + if remainingPath != "" { + fullPath += "." + remainingPath } + var setErr error + jsonStr, setErr = v.setValueWithSjson(jsonStr, fullPath, value) + if setErr != nil { + err = setErr + return false + } + return true + }) + if err != nil { return nil, errors.WithMessage(err, "failed to set config value of array child") } + } else if result.IsObject() { + result.ForEach(func(key, val gjson.Result) bool { + fullPath := basePath + "." + key.String() + if remainingPath != "" { + fullPath += "." + remainingPath + } + var setErr error + jsonStr, setErr = v.setValueWithSjson(jsonStr, fullPath, value) + if setErr != nil { + err = setErr + return false + } + return true + }) + if err != nil { + return nil, errors.WithMessage(err, "failed to set config value of object child") + } } continue } - if err := v.SetAtPathway(parsed, v.Match, value); err != nil { - if errors.Is(err, gabs.ErrNotFound) { + var setErr error + jsonStr, setErr = v.setValueWithSjson(jsonStr, v.Match, value) + if setErr != nil { + if strings.Contains(setErr.Error(), "path not found") { continue } - return nil, errors.WithMessage(err, "unable to set config value at pathway: "+v.Match) + return nil, errors.WithMessage(setErr, "unable to set config value at pathway: "+v.Match) } } - return parsed, nil + return []byte(jsonStr), nil } -// Regex used to check if there is an array element present in the given pathway by looking for something -// along the lines of "something[1]" or "something[1].nestedvalue" as the path. -var checkForArrayElement = regexp.MustCompile(`^([^\[\]]+)\[([\d]+)](\..+)?$`) - -// Attempt to set the value of the path depending on if it is an array or not. Gabs cannot handle array -// values as "something[1]" but can parse them just fine. This is basically just overly complex code -// to handle that edge case and ensure the value gets set correctly. -// -// Bless thee who has to touch these most unholy waters. -func setValueAtPath(c *gabs.Container, path string, value interface{}) error { - var err error - - matches := checkForArrayElement.FindStringSubmatch(path) - - // Check if we are **NOT** updating an array element. - if len(matches) < 3 { - _, err = c.SetP(value, path) - return err - } +func (cfr *ConfigurationFileReplacement) setValueWithSjson(jsonStr string, path string, value string) (string, error) { + if cfr.IfValue != "" { + // Check if we are replacing instead of overwriting. + if strings.HasPrefix(cfr.IfValue, "regex:") { + result := gjson.Get(jsonStr, path) + if !result.Exists() { + return jsonStr, nil + } - i, _ := strconv.Atoi(matches[2]) - // Find the array element "i" or try to create it if "i" is equal to 0 and is not found - // at the given path. - ct, err := c.ArrayElementP(i, matches[1]) - if err != nil { - if i != 0 || (!errors.Is(err, gabs.ErrNotArray) && !errors.Is(err, gabs.ErrNotFound)) { - return errors.WithMessage(err, "error while parsing array element at path") - } + r, err := regexp.Compile(strings.TrimPrefix(cfr.IfValue, "regex:")) + if err != nil { + log.WithFields(log.Fields{"if_value": strings.TrimPrefix(cfr.IfValue, "regex:"), "error": err}). + Warn("configuration if_value using invalid regexp, cannot perform replacement") + return jsonStr, nil + } - t := make([]interface{}, 1) - // If the length of matches is 4 it means we're trying to access an object down in this array - // key, so make sure we generate the array as an array of objects, and not just a generic nil - // array. - if len(matches) == 4 { - t = []interface{}{map[string]interface{}{}} + v := result.String() + if r.MatchString(v) { + newValue := r.ReplaceAllString(v, value) + return sjson.Set(jsonStr, path, newValue) + } + return jsonStr, nil } - // If the error is because this isn't an array or isn't found go ahead and create the array with - // an empty object if we have additional things to set on the array, or just an empty array type - // if there is not an object structure detected (no matches[3] available). - if _, err = c.SetP(t, matches[1]); err != nil { - return errors.WithMessage(err, "failed to create empty array for missing element") + result := gjson.Get(jsonStr, path) + if !result.Exists() { + return jsonStr, nil } - - // Set our cursor to be the array element we expect, which in this case is just the first element - // since we won't run this code unless the array element is 0. There is too much complexity in trying - // to match additional elements. In those cases the server will just have to be rebooted or something. - ct, err = c.ArrayElementP(0, matches[1]) - if err != nil { - return errors.WithMessage(err, "failed to find array element at path") + if result.String() != cfr.IfValue { + return jsonStr, nil } } - // Try to set the value. If the path does not exist an error will be raised to the caller which will - // then check if the error is because the path is missing. In those cases we just ignore the error since - // we don't want to do anything specifically when that happens. - // - // If there are four matches in the regex it means that we managed to also match a trailing pathway - // for the key, which should be found in the given array key item and modified further. - if len(matches) == 4 { - _, err = ct.SetP(value, strings.TrimPrefix(matches[3], ".")) + var setValue interface{} + if cfr.ReplaceWith.Type() == jsonparser.Boolean { + v, _ := strconv.ParseBool(value) + setValue = v + } else if v, err := strconv.Atoi(value); err == nil { + setValue = v } else { - _, err = ct.Set(value) - } - - if err != nil { - return errors.WithMessage(err, "failed to set value at config path: "+path) - } - - return nil -} - -// Sets the value at a specific pathway, but checks if we were looking for a specific -// value or not before doing it. -func (cfr *ConfigurationFileReplacement) SetAtPathway(c *gabs.Container, path string, value string) error { - if cfr.IfValue == "" { - return setValueAtPath(c, path, cfr.getKeyValue(value)) - } - - // Check if we are replacing instead of overwriting. - if strings.HasPrefix(cfr.IfValue, "regex:") { - // Doing a regex replacement requires an existing value. - // TODO: Do we try passing an empty string to the regex? - if c.ExistsP(path) { - return gabs.ErrNotFound - } - - r, err := regexp.Compile(strings.TrimPrefix(cfr.IfValue, "regex:")) - if err != nil { - log.WithFields(log.Fields{"if_value": strings.TrimPrefix(cfr.IfValue, "regex:"), "error": err}). - Warn("configuration if_value using invalid regexp, cannot perform replacement") - return nil - } - - v := strings.Trim(c.Path(path).String(), "\"") - if r.Match([]byte(v)) { - return setValueAtPath(c, path, r.ReplaceAllString(v, value)) - } - return nil - } - - if c.ExistsP(path) && !bytes.Equal(c.Bytes(), []byte(cfr.IfValue)) { - return nil + setValue = value } - return setValueAtPath(c, path, cfr.getKeyValue(value)) + return sjson.Set(jsonStr, path, setValue) } // Looks up a configuration value on the Daemon given a dot-notated syntax. diff --git a/parser/parser.go b/parser/parser.go index 45aaa67f..d83717e1 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -14,6 +14,7 @@ import ( "github.com/goccy/go-json" "github.com/icza/dyno" "github.com/magiconair/properties" + "github.com/tidwall/pretty" "gopkg.in/ini.v1" "gopkg.in/yaml.v3" @@ -414,8 +415,14 @@ func (f *ConfigurationFile) parseJsonFile(file ufs.File) error { return err } - // Write the data to the file. - if _, err := io.Copy(file, bytes.NewReader(data.BytesIndent("", " "))); err != nil { + prettified := pretty.PrettyOptions(data, &pretty.Options{ + Width: 80, + Prefix: "", + Indent: " ", + SortKeys: false, + }) + + if _, err := io.Copy(file, bytes.NewReader(prettified)); err != nil { return errors.Wrap(err, "parser: failed to write properties file to disk") } return nil @@ -435,8 +442,8 @@ func (f *ConfigurationFile) parseYamlFile(file ufs.File) error { } // Unmarshal the yaml data into a JSON interface such that we can work with - // any arbitrary data structure. If we don't do this, I can't use gabs which - // makes working with unknown JSON significantly easier. + // any arbitrary data structure. This allows us to use gjson/sjson for + // working with unknown JSON significantly easier. jsonBytes, err := json.Marshal(dyno.ConvertMapI2MapS(i)) if err != nil { return err @@ -449,8 +456,12 @@ func (f *ConfigurationFile) parseYamlFile(file ufs.File) error { return err } - // Remarshal the JSON into YAML format before saving it back to the disk. - marshaled, err := yaml.Marshal(data.Data()) + var jsonData interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return err + } + + marshaled, err := yaml.Marshal(jsonData) if err != nil { return err }