diff --git a/.gitignore b/.gitignore index 94000406..d56f441a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ agent-config.test.yaml infisical-merge test/infisical-merge .DS_Store + + +/agent-testing \ No newline at end of file diff --git a/go.mod b/go.mod index 46449016..015be2a1 100644 --- a/go.mod +++ b/go.mod @@ -72,9 +72,14 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 // indirect github.com/aws/smithy-go v1.20.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgraph-io/badger/v3 v3.2103.5 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -91,8 +96,11 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/go.sum b/go.sum index a0d86980..c3f0e0fb 100644 --- a/go.sum +++ b/go.sum @@ -63,10 +63,12 @@ github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+ github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= @@ -105,6 +107,11 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -120,8 +127,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -135,6 +146,13 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= @@ -155,6 +173,7 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -200,6 +219,8 @@ github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14j github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -233,8 +254,12 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -319,6 +344,7 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/infisical/go-sdk v0.5.99 h1:trvn7JhKYuSzDkc44h+yqToVjclkrRyP42t315k5kEE= @@ -344,6 +370,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= @@ -358,6 +385,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -379,6 +407,7 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -425,6 +454,7 @@ github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlR github.com/oracle/oci-go-sdk/v65 v65.95.2 h1:0HJ0AgpLydp/DtvYrF2d4str2BjXOVAeNbuW7E07g94= github.com/oracle/oci-go-sdk/v65 v65.95.2/go.mod h1:u6XRPsw9tPziBh76K7GrrRXPa8P8W3BQeqJ6ZZt9VLA= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -471,6 +501,7 @@ github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -484,17 +515,25 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -521,6 +560,7 @@ github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZB github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/wasilibs/go-re2 v1.10.0 h1:vQZEBYZOCA9jdBMmrO4+CvqyCj0x4OomXTJ4a5/urQ0= github.com/wasilibs/go-re2 v1.10.0/go.mod h1:k+5XqO2bCJS+QpGOnqugyfwC04nw0jaglmjrrkG8U6o= @@ -533,6 +573,7 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= @@ -588,6 +629,7 @@ go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -719,6 +761,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -764,6 +807,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -975,6 +1019,7 @@ google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwl google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/main.go b/main.go index 75152ffc..1b8a30cb 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,10 @@ Copyright (c) 2023 Infisical Inc. package main import ( + "fmt" "os" + "strings" + "time" "github.com/Infisical/infisical-merge/packages/cmd" "github.com/rs/zerolog" @@ -12,6 +15,46 @@ import ( ) func main() { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + + // very annoying but zerolog doesn't allow us to change one color without changing all of them + // these are the default colors for each level, except for warn + levelColors := map[string]string{ + "trace": "\033[35m", // magenta + "debug": "\033[33m", // yellow + "info": "\033[32m", // green + "warn": "\033[33m", // yellow (this one is custom, the default is red \033[31m) + "error": "\033[31m", // red + "fatal": "\033[31m", // red + "panic": "\033[31m", // red + } + + // map full level names to abbreviated forms (default zerolog behavior) + // see consoleDefaultFormatLevel, in zerolog for example + levelAbbrev := map[string]string{ + "trace": "TRC", + "debug": "DBG", + "info": "INF", + "warn": "WRN", + "error": "ERR", + "fatal": "FTL", + "panic": "PNC", + } + + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stderr, + TimeFormat: time.RFC3339, + FormatLevel: func(i interface{}) string { + level := fmt.Sprintf("%s", i) + color := levelColors[level] + if color == "" { + color = "\033[0m" // no color for unknown levels + } + abbrev := levelAbbrev[level] + if abbrev == "" { + abbrev = strings.ToUpper(level) // fallback to uppercase if unknown + } + return color + abbrev + "\033[0m" + }, + }) cmd.Execute() } diff --git a/packages/api/api.go b/packages/api/api.go index c4eb7586..2517cec4 100644 --- a/packages/api/api.go +++ b/packages/api/api.go @@ -2,6 +2,7 @@ package api import ( "encoding/base64" + "errors" "fmt" "net/http" "strings" @@ -54,6 +55,8 @@ const ( operationCallInstanceRelayHeartBeat = "CallInstanceRelayHeartBeat" ) +var ErrNotFound = errors.New("resource not found") + func CallGetEncryptedWorkspaceKey(httpClient *resty.Client, request GetEncryptedWorkspaceKeyRequest) (GetEncryptedWorkspaceKeyResponse, error) { endpoint := fmt.Sprintf("%v/v2/workspace/%v/encrypted-key", config.INFISICAL_URL, request.WorkspaceId) var result GetEncryptedWorkspaceKeyResponse @@ -572,6 +575,31 @@ func CallCreateDynamicSecretLeaseV1(httpClient *resty.Client, request CreateDyna return createDynamicSecretLeaseResponse, nil } +func CallGetDynamicSecretLeaseV1(httpClient *resty.Client, request GetDynamicSecretLeaseV1Request) (GetDynamicSecretLeaseV1Response, error) { + var getDynamicSecretLeaseResponse GetDynamicSecretLeaseV1Response + response, err := httpClient. + R(). + SetResult(&getDynamicSecretLeaseResponse). + SetHeader("User-Agent", USER_AGENT). + SetQueryParam("environmentSlug", request.Environment). + SetQueryParam("projectSlug", request.ProjectSlug). + SetQueryParam("secretPath", request.SecretPath). + Get(fmt.Sprintf("%v/v1/dynamic-secrets/leases/%s", config.INFISICAL_URL, request.LeaseID)) + + if err != nil { + return GetDynamicSecretLeaseV1Response{}, fmt.Errorf("CallGetDynamicSecretLeaseV1: Unable to complete api request [err=%w]", err) + } + + if response.IsError() { + if response.StatusCode() == http.StatusNotFound { + return GetDynamicSecretLeaseV1Response{}, ErrNotFound + } + return GetDynamicSecretLeaseV1Response{}, fmt.Errorf("CallGetDynamicSecretLeaseV1: Unsuccessful response [status-code=%v] [response=%v]", response.StatusCode(), response.String()) + } + + return getDynamicSecretLeaseResponse, nil +} + func CallCreateRawSecretsV3(httpClient *resty.Client, request CreateRawSecretV3Request) error { response, err := httpClient. R(). diff --git a/packages/api/model.go b/packages/api/model.go index 07b35f67..fcfa81b3 100644 --- a/packages/api/model.go +++ b/packages/api/model.go @@ -585,6 +585,21 @@ type CreateDynamicSecretLeaseV1Response struct { Data map[string]interface{} `json:"data"` } +type GetDynamicSecretLeaseV1Request struct { + LeaseID string + Environment string + ProjectSlug string + SecretPath string +} + +type GetDynamicSecretLeaseV1Response struct { + Lease struct { + Id string `json:"id"` + ExpireAt time.Time `json:"expireAt"` + } `json:"lease"` + DynamicSecret models.DynamicSecret `json:"dynamicSecret"` +} + type GetLoginV3Request struct { Email string `json:"email"` Password string `json:"password"` diff --git a/packages/cmd/agent.go b/packages/cmd/agent.go index 2d7cb4be..25812798 100644 --- a/packages/cmd/agent.go +++ b/packages/cmd/agent.go @@ -6,10 +6,13 @@ package cmd import ( "bytes" "context" + "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io/ioutil" + "net/http" "os" "os/exec" "os/signal" @@ -18,10 +21,12 @@ import ( "slices" "strings" "sync" + "sync/atomic" "syscall" "text/template" "time" + "github.com/dgraph-io/badger/v3" infisicalSdk "github.com/infisical/go-sdk" "github.com/rs/zerolog/log" "gopkg.in/yaml.v2" @@ -30,18 +35,51 @@ import ( "github.com/Infisical/infisical-merge/packages/config" "github.com/Infisical/infisical-merge/packages/models" "github.com/Infisical/infisical-merge/packages/util" + "github.com/Infisical/infisical-merge/packages/util/cache" "github.com/spf13/cobra" ) const DEFAULT_INFISICAL_CLOUD_URL = "https://app.infisical.com" +const CACHE_TYPE_KUBERNETES = "kubernetes" + +const DYNAMIC_SECRET_LEASE_TEMPLATE = "dynamic-secret-lease-%s-%s-%s-%s-%d" + // duration to reduce from expiry of dynamic leases so that it gets triggered before expiry const DYNAMIC_SECRET_PRUNE_EXPIRE_BUFFER = -15 +// duration remove leases from the cache before they expire when the agent is first started with existing leases in the cache. +// if a lease is expired, or expires in 30 seconds or less, it will be deleted from the cache and a new lease will be created. +var CACHE_LEASE_EXPIRE_BUFFER = 30 * time.Second + +type PersistentCacheConfig struct { + Type string `yaml:"type"` // file or kubernetes + ServiceAccountTokenPath string `yaml:"service-account-token-path"` // relevant if type is kubernetes + Path string `yaml:"path"` // where to store the cache +} + +type CacheConfig struct { + Persistent *PersistentCacheConfig `yaml:"persistent,omitempty"` +} + +type DecryptedCache struct { + Type string `json:"type"` // currently only "access_token" is supported + AccessToken string `json:"access_token"` +} + +type CacheManager struct { + cacheConfig *CacheConfig + cacheStorage *cache.EncryptedStorage + + IsEnabled bool + DecryptedCache DecryptedCache +} + type Config struct { Infisical InfisicalConfig `yaml:"infisical"` Auth AuthConfig `yaml:"auth"` Sinks []Sink `yaml:"sinks"` + Cache CacheConfig `yaml:"cache,omitempty"` Templates []Template `yaml:"templates"` } @@ -128,12 +166,272 @@ type DynamicSecretLease struct { Slug string ProjectSlug string Data map[string]interface{} - TemplateIDs []int + TemplateID int +} + +func (c *CacheManager) WriteToCache(key string, value interface{}, ttl *time.Duration) error { + + if !c.IsEnabled { + return nil + } + + var err error + + if ttl != nil { + if *ttl <= 0 { + return fmt.Errorf("ttl must be greater than 0") + } + err = c.cacheStorage.SetWithTTL(key, value, *ttl) + } else { + err = c.cacheStorage.Set(key, value) + } + if err != nil && !errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("unable to write to cache: %v", err) + } + return nil +} + +func (c *CacheManager) GetAllCacheEntries() (map[string]interface{}, error) { + + if c.cacheStorage == nil || !c.IsEnabled { + return nil, nil + } + + response, err := c.cacheStorage.GetAll() + if err != nil { + return nil, fmt.Errorf("unable to get all cache keys: %v", err) + } + return response, nil +} + +func (c *CacheManager) ReadFromCache(key string, destination interface{}) error { + err := c.cacheStorage.Get(key, destination) + if err != nil && !errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("unable to read from cache: %v", err) + } + + return nil +} + +func (c *CacheManager) DeleteFromCache(key string) error { + if !c.IsEnabled { + return nil + } + err := c.cacheStorage.Delete(key) + if err != nil && !errors.Is(err, badger.ErrKeyNotFound) { + return fmt.Errorf("unable to delete from cache: %v", err) + } + return nil +} + +func NewCacheManager(ctx context.Context, cacheConfig *CacheConfig) (*CacheManager, error) { + + if cacheConfig == nil || cacheConfig.Persistent == nil { + log.Info().Msg("caching is disabled, continuing without caching.") + return &CacheManager{ + IsEnabled: false, + DecryptedCache: DecryptedCache{}, + cacheConfig: cacheConfig, + }, nil + } + + if cacheConfig.Persistent.Type != CACHE_TYPE_KUBERNETES { + return &CacheManager{}, fmt.Errorf("unsupported cache type: %s", cacheConfig.Persistent.Type) + } + + // try to read the service account token file + serviceAccountToken, err := ReadFile(cacheConfig.Persistent.ServiceAccountTokenPath) + if err != nil || len(serviceAccountToken) == 0 { + return &CacheManager{}, fmt.Errorf("unable to read service account token: %v. Please ensure the file exists and is not empty", err) + } + + encryptionKey := sha256.Sum256(serviceAccountToken) + + cacheStorage, err := cache.NewEncryptedStorage(cache.EncryptedStorageOptions{ + DBPath: cacheConfig.Persistent.Path, + EncryptionKey: encryptionKey, + InMemory: false, + }) + + go cacheStorage.StartPeriodicGarbageCollection(ctx) + + if err != nil { + return nil, fmt.Errorf("unable to create cache storage: %v", err) + } + + return &CacheManager{ + IsEnabled: true, + cacheConfig: cacheConfig, + cacheStorage: cacheStorage, + }, nil } type DynamicSecretLeaseManager struct { - leases []DynamicSecretLease - mutex sync.Mutex + leases []DynamicSecretLease + mutex sync.Mutex + cacheManager *CacheManager +} + +func (d *DynamicSecretLeaseManager) WriteLeaseToCache(lease *DynamicSecretLease, templateId int) { + + if d.cacheManager == nil || !d.cacheManager.IsEnabled { + return + } + + if lease == nil { + return + } + + cacheKey := fmt.Sprintf( + DYNAMIC_SECRET_LEASE_TEMPLATE, + lease.ProjectSlug, + lease.Environment, + lease.SecretPath, + lease.Slug, + templateId, + ) + + ttl := lease.ExpireAt.Sub(time.Now()) + + log.Info().Msgf("[cache]: writing dynamic secret lease to cache: [cache-key=%s] [entry-ttl=%s]", cacheKey, ttl.String()) + + if err := d.cacheManager.WriteToCache(cacheKey, lease, &ttl); err != nil { + log.Error().Msgf("[cache]: unable to write dynamic secret lease to cache because %v", err) + } else { + log.Info().Msgf("[cache]: dynamic secret lease written to cache: %s", cacheKey) + } +} + +func (d *DynamicSecretLeaseManager) ReadLeaseFromCache(projectSlug, environment, secretPath, slug string, templateId int) *DynamicSecretLease { + + if d.cacheManager == nil || !d.cacheManager.IsEnabled { + return nil + } + + cacheKey := fmt.Sprintf(DYNAMIC_SECRET_LEASE_TEMPLATE, projectSlug, environment, secretPath, slug, templateId) + var lease *DynamicSecretLease + err := d.cacheManager.ReadFromCache(cacheKey, &lease) + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return nil + } + log.Error().Msgf("[cache]: unable to read dynamic secret lease from cache because %v", err) + return nil + } + return lease +} + +func (d *DynamicSecretLeaseManager) DeleteLeaseFromCache(projectSlug, environment, secretPath, slug string, templateId int) error { + if d.cacheManager == nil || !d.cacheManager.IsEnabled { + return nil + } + + cacheKey := fmt.Sprintf(DYNAMIC_SECRET_LEASE_TEMPLATE, projectSlug, environment, secretPath, slug, templateId) + err := d.cacheManager.DeleteFromCache(cacheKey) + if err != nil { + return fmt.Errorf("unable to delete lease from cache: %v", err) + } + return nil +} + +func (d *DynamicSecretLeaseManager) DeleteUnusedLeasesFromCache() error { + + if d.cacheManager.IsEnabled { + log.Info().Msgf("[cache]: deleting unused dynamic secret leases from cache") + } + + d.mutex.Lock() + defer d.mutex.Unlock() + + allCacheKeys, err := d.cacheManager.GetAllCacheEntries() + + if err != nil { + return fmt.Errorf("unable to get all cache entries: %v", err) + } + + if allCacheKeys == nil { + log.Debug().Msgf("[cache]: no cache entries found") + return nil + } + + var cachedLeases []DynamicSecretLease + for cacheKey, leaseData := range allCacheKeys { + if strings.HasPrefix(cacheKey, "dynamic-secret-lease-") { + // Marshal back to JSON and unmarshal into the correct type + jsonData, err := json.Marshal(leaseData) + if err != nil { + log.Warn().Msgf("[cache]: failed to marshal cached lease data for key %s: %v", cacheKey, err) + continue + } + + var lease DynamicSecretLease + if err := json.Unmarshal(jsonData, &lease); err != nil { + log.Warn().Msgf("[cache]: failed to unmarshal cached lease data for key %s: %v", cacheKey, err) + continue + } + + cachedLeases = append(cachedLeases, lease) + } + } + + log.Debug().Msgf("[cache]: found %d cached leases", len(cachedLeases)) + log.Debug().Msgf("[cache]: current active leases count: %d", len(d.leases)) + + // now we need to check if any of the cached leases are not in the d.leases list. If they are not, we need to delete them from the cache. + for _, cachedLease := range cachedLeases { + log.Debug().Msgf( + "[cache]: checking cached lease: [project=%s], [env=%s], [path=%s], [slug=%s], [template-id=%d]", + cachedLease.ProjectSlug, + cachedLease.Environment, + cachedLease.SecretPath, + cachedLease.Slug, + cachedLease.TemplateID, + ) + + // check if a lease with the same configuration exists (not comparing LeaseID since that changes on refresh) + found := slices.ContainsFunc(d.leases, func(s DynamicSecretLease) bool { + match := s.ProjectSlug == cachedLease.ProjectSlug && + s.Environment == cachedLease.Environment && + s.SecretPath == cachedLease.SecretPath && + s.Slug == cachedLease.Slug && + s.TemplateID == cachedLease.TemplateID + + if match { + log.Debug().Msgf("[cache]: found matching active lease: [project=%s], [env=%s], [path=%s], [slug=%s], [template-id=%d]", + s.ProjectSlug, + s.Environment, + s.SecretPath, + s.Slug, + s.TemplateID, + ) + } + return match + }) + + if !found { + log.Info().Msgf( + "[cache]: no matching active lease found, deleting cached lease: [lease-id=%s], [project=%s], [env=%s], [path=%s], [slug=%s]", + cachedLease.LeaseID, + cachedLease.ProjectSlug, + cachedLease.Environment, + cachedLease.SecretPath, + cachedLease.Slug, + ) + + if err := d.DeleteLeaseFromCache( + cachedLease.ProjectSlug, + cachedLease.Environment, + cachedLease.SecretPath, + cachedLease.Slug, + cachedLease.TemplateID, + ); err != nil { + log.Warn().Msgf("[cache]: unable to delete lease from cache: %v", err) + } + } + } + + return nil + } func (d *DynamicSecretLeaseManager) Prune() { @@ -141,13 +439,19 @@ func (d *DynamicSecretLeaseManager) Prune() { defer d.mutex.Unlock() d.leases = slices.DeleteFunc(d.leases, func(s DynamicSecretLease) bool { - return time.Now().After(s.ExpireAt.Add(DYNAMIC_SECRET_PRUNE_EXPIRE_BUFFER * time.Second)) + shouldDelete := time.Now().After(s.ExpireAt.Add(DYNAMIC_SECRET_PRUNE_EXPIRE_BUFFER * time.Second)) + + if shouldDelete { + if err := d.DeleteLeaseFromCache(s.ProjectSlug, s.Environment, s.SecretPath, s.Slug, s.TemplateID); err != nil { + log.Warn().Msgf("[cache]: unable to delete lease from cache: %v", err) + } + } + return shouldDelete }) } -func (d *DynamicSecretLeaseManager) Append(lease DynamicSecretLease) { - d.mutex.Lock() - defer d.mutex.Unlock() +// appendUnsafe can be used if you already hold the lock +func (d *DynamicSecretLeaseManager) appendUnsafe(lease DynamicSecretLease) { index := slices.IndexFunc(d.leases, func(s DynamicSecretLease) bool { if lease.SecretPath == s.SecretPath && lease.Environment == s.Environment && lease.ProjectSlug == s.ProjectSlug && lease.Slug == s.Slug && lease.LeaseID == s.LeaseID { @@ -157,10 +461,22 @@ func (d *DynamicSecretLeaseManager) Append(lease DynamicSecretLease) { }) if index != -1 { - d.leases[index].TemplateIDs = append(d.leases[index].TemplateIDs, lease.TemplateIDs...) + d.leases[index].TemplateID = lease.TemplateID return } + d.leases = append(d.leases, lease) + + d.WriteLeaseToCache(&lease, lease.TemplateID) + +} + +func (d *DynamicSecretLeaseManager) Append(lease DynamicSecretLease) { + + d.mutex.Lock() + defer d.mutex.Unlock() + + d.appendUnsafe(lease) } func (d *DynamicSecretLeaseManager) RegisterTemplate(projectSlug, environment, secretPath, slug string, templateId int) { @@ -168,33 +484,101 @@ func (d *DynamicSecretLeaseManager) RegisterTemplate(projectSlug, environment, s defer d.mutex.Unlock() index := slices.IndexFunc(d.leases, func(lease DynamicSecretLease) bool { - if lease.SecretPath == secretPath && lease.Environment == environment && lease.ProjectSlug == projectSlug && lease.Slug == slug && slices.Contains(lease.TemplateIDs, templateId) { + if lease.SecretPath == secretPath && lease.Environment == environment && lease.ProjectSlug == projectSlug && lease.Slug == slug && lease.TemplateID == templateId { return true } + return false }) if index != -1 { - d.leases[index].TemplateIDs = append(d.leases[index].TemplateIDs, templateId) + d.leases[index].TemplateID = templateId } } -func (d *DynamicSecretLeaseManager) GetLease(projectSlug, environment, secretPath, slug string, templateId int) *DynamicSecretLease { +func (d *DynamicSecretLeaseManager) GetLease(accessToken, projectSlug, environment, secretPath, slug string, templateId int) *DynamicSecretLease { d.mutex.Lock() defer d.mutex.Unlock() + // first try to get from in-memory storage + for _, lease := range d.leases { - if lease.SecretPath == secretPath && lease.Environment == environment && lease.ProjectSlug == projectSlug && lease.Slug == slug && slices.Contains(lease.TemplateIDs, templateId) { + if lease.SecretPath == secretPath && lease.Environment == environment && lease.ProjectSlug == projectSlug && lease.Slug == slug && lease.TemplateID == templateId { + log.Debug().Msgf("[cache]: lease found in in-memory storage: [project=%s], [env=%s], [path=%s], [slug=%s], [template-id=%d]", projectSlug, environment, secretPath, slug, templateId) return &lease } } + // if no lease is found in in-memory storage, try to get from cache + + log.Info().Msgf("[cache]: no lease found, fetching from cache") + leaseFromCache := d.ReadLeaseFromCache(projectSlug, environment, secretPath, slug, templateId) + log.Debug().Msgf("[cache]: lease from cache: %+v", leaseFromCache) + + if leaseFromCache != nil { + + // try to get the lease from the API + + // ? question(daniel): should we trust the cache more, and avoid calling the API to see if the lease still exists? + // ? we do this to ensure that the lease wasn't deleted while the agent was not running + + dynamicSecretLease, err := util.GetDynamicSecretLease(accessToken, leaseFromCache.ProjectSlug, leaseFromCache.Environment, leaseFromCache.SecretPath, leaseFromCache.LeaseID) + if err != nil { + + log.Warn().Msgf("[cache]: error: %+v", err) + + // lease not found in API, delete it from cache and return nil + if errors.Is(err, api.ErrNotFound) { + log.Warn().Msgf("dynamic secret lease does not exist, deleting from cache: [lease-id=%s]", leaseFromCache.LeaseID) + if err := d.DeleteLeaseFromCache(leaseFromCache.ProjectSlug, leaseFromCache.Environment, leaseFromCache.SecretPath, leaseFromCache.Slug, leaseFromCache.TemplateID); err != nil { + log.Warn().Msgf("[cache]: unable to delete lease from cache: %v", err) + } + + return nil + } + + // lease is found in cache but not in the the API, and the API returned a non 404-error. We should attempt to revoke it + // at this point we know that we should be able to reach the API because we've done authentication successfully + log.Warn().Msgf("unable to get dynamic secret lease from API. Revoking lease from cache: [lease-id=%s]", leaseFromCache.LeaseID) + if err := d.DeleteLeaseFromCache(leaseFromCache.ProjectSlug, leaseFromCache.Environment, leaseFromCache.SecretPath, leaseFromCache.Slug, leaseFromCache.TemplateID); err != nil { + log.Warn().Msgf("[cache]: unable to delete lease from cache: %v", err) + } + + if err := revokeDynamicSecretLease(accessToken, leaseFromCache.ProjectSlug, leaseFromCache.Environment, leaseFromCache.SecretPath, leaseFromCache.LeaseID); err != nil { + log.Warn().Msgf("unable to revoke dynamic secret lease %s: %v", leaseFromCache.LeaseID, err) + return nil + } + + return nil + } + + // lease is expired or about to expire, delete from cache and attempt to revoke it + if dynamicSecretLease.Lease.ExpireAt.Before(time.Now().Add(CACHE_LEASE_EXPIRE_BUFFER)) { + log.Warn().Msgf("dynamic secret lease is expired or about to expire, deleting from cache: [lease-id=%s]", leaseFromCache.LeaseID) + if err := d.DeleteLeaseFromCache(leaseFromCache.ProjectSlug, leaseFromCache.Environment, leaseFromCache.SecretPath, leaseFromCache.Slug, leaseFromCache.TemplateID); err != nil { + log.Warn().Msgf("[cache]: unable to delete lease from cache: %v", err) + } + + if err := revokeDynamicSecretLease(accessToken, leaseFromCache.ProjectSlug, leaseFromCache.Environment, leaseFromCache.SecretPath, leaseFromCache.LeaseID); err != nil { + log.Warn().Msgf("unable to revoke expired dynamic secret lease %s: %v. Non-critical, the lease is already expired or will expire automatically within the next 2 minutes.", leaseFromCache.LeaseID, err) + return nil + } + + return nil + } + + // we call appendUnsafe because we already hold the lock, and if we call Append directly we'll get a deadlock + d.appendUnsafe(*leaseFromCache) + + return leaseFromCache + } + return nil } // for a given template find the first expiring lease // The bool indicates whether it contains valid expiry list -func (d *DynamicSecretLeaseManager) GetFirstExpiringLeaseTime(templateId int) (time.Time, bool) { +func (d *DynamicSecretLeaseManager) GetFirstExpiringLeaseTime() (time.Time, bool) { d.mutex.Lock() defer d.mutex.Unlock() @@ -215,8 +599,10 @@ func (d *DynamicSecretLeaseManager) GetFirstExpiringLeaseTime(templateId int) (t return firstExpiry, true } -func NewDynamicSecretLeaseManager(sigChan chan os.Signal) *DynamicSecretLeaseManager { - manager := &DynamicSecretLeaseManager{} +func NewDynamicSecretLeaseManager(sigChan chan os.Signal, cacheManager *CacheManager) *DynamicSecretLeaseManager { + manager := &DynamicSecretLeaseManager{ + cacheManager: cacheManager, + } return manager } @@ -289,15 +675,7 @@ func ParseAuthConfig(authConfigFile []byte, destination interface{}) error { } func ParseAgentConfig(configFile []byte) (*Config, error) { - var rawConfig struct { - Infisical InfisicalConfig `yaml:"infisical"` - Auth struct { - Type string `yaml:"type"` - Config map[string]interface{} `yaml:"config"` - } `yaml:"auth"` - Sinks []Sink `yaml:"sinks"` - Templates []Template `yaml:"templates"` - } + var rawConfig Config if err := yaml.Unmarshal(configFile, &rawConfig); err != nil { return nil, err @@ -308,21 +686,17 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { rawConfig.Infisical.Address = DEFAULT_INFISICAL_CLOUD_URL } + if rawConfig.Cache.Persistent != nil && rawConfig.Cache.Persistent.Type == CACHE_TYPE_KUBERNETES { + if rawConfig.Cache.Persistent.ServiceAccountTokenPath == "" { + rawConfig.Cache.Persistent.ServiceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + } + config.INFISICAL_URL = util.AppendAPIEndpoint(rawConfig.Infisical.Address) log.Info().Msgf("Infisical instance address set to %s", rawConfig.Infisical.Address) - config := &Config{ - Infisical: rawConfig.Infisical, - Auth: AuthConfig{ - Type: rawConfig.Auth.Type, - Config: rawConfig.Auth.Config, - }, - Sinks: rawConfig.Sinks, - Templates: rawConfig.Templates, - } - - return config, nil + return &rawConfig, nil } type secretArguments struct { @@ -381,6 +755,7 @@ func getSingleSecretTemplateFunction(accessToken string, existingEtag string, cu } func dynamicSecretTemplateFunction(accessToken string, dynamicSecretManager *DynamicSecretLeaseManager, templateId int, currentEtag *string) func(...string) (map[string]interface{}, error) { + return func(args ...string) (map[string]interface{}, error) { argLength := len(args) if argLength != 4 && argLength != 5 { @@ -391,12 +766,15 @@ func dynamicSecretTemplateFunction(accessToken string, dynamicSecretManager *Dyn if argLength == 5 { ttl = args[4] } - dynamicSecretData := dynamicSecretManager.GetLease(projectSlug, envSlug, secretPath, slug, templateId) + dynamicSecretData := dynamicSecretManager.GetLease(accessToken, projectSlug, envSlug, secretPath, slug, templateId) + + // if a lease is found (either in memory or in cache), we register the template and return the data if dynamicSecretData != nil { dynamicSecretManager.RegisterTemplate(projectSlug, envSlug, secretPath, slug, templateId) return dynamicSecretData.Data, nil } + // if there's no lease (either in memory or in cache), we create a new lease res, err := util.CreateDynamicSecretLease(accessToken, projectSlug, envSlug, secretPath, slug, ttl) if err != nil { return nil, err @@ -405,13 +783,14 @@ func dynamicSecretTemplateFunction(accessToken string, dynamicSecretManager *Dyn // we set an arbitrary etag to ensure that the template is re-rendered when a new lease is created *currentEtag = util.GenerateRandomString(32) - dynamicSecretManager.Append(DynamicSecretLease{LeaseID: res.Lease.Id, ExpireAt: res.Lease.ExpireAt, Environment: envSlug, SecretPath: secretPath, Slug: slug, ProjectSlug: projectSlug, Data: res.Data, TemplateIDs: []int{templateId}}) + dynamicSecretManager.Append(DynamicSecretLease{LeaseID: res.Lease.Id, ExpireAt: res.Lease.ExpireAt, Environment: envSlug, SecretPath: secretPath, Slug: slug, ProjectSlug: projectSlug, Data: res.Data, TemplateID: templateId}) return res.Data, nil } } func ProcessTemplate(templateId int, templatePath string, data interface{}, accessToken string, existingEtag string, currentEtag *string, dynamicSecretManager *DynamicSecretLeaseManager) (*bytes.Buffer, error) { + // custom template function to fetch secrets from Infisical secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) dynamicSecretFunction := dynamicSecretTemplateFunction(accessToken, dynamicSecretManager, templateId, currentEtag) @@ -443,7 +822,7 @@ func ProcessTemplate(templateId int, templatePath string, data interface{}, acce return &buf, nil } -func ProcessBase64Template(templateId int, encodedTemplate string, data interface{}, accessToken string, existingEtag string, currentEtag *string, dynamicSecretLeaser *DynamicSecretLeaseManager) (*bytes.Buffer, error) { +func ProcessBase64Template(templateId int, encodedTemplate string, data interface{}, accessToken string, existingEtag string, currentEtag *string, dynamicSecretLeaseManager *DynamicSecretLeaseManager) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical decoded, err := base64.StdEncoding.DecodeString(encodedTemplate) if err != nil { @@ -453,7 +832,7 @@ func ProcessBase64Template(templateId int, encodedTemplate string, data interfac templateString := string(decoded) secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) // TODO: Fix this - dynamicSecretFunction := dynamicSecretTemplateFunction(accessToken, dynamicSecretLeaser, templateId, currentEtag) + dynamicSecretFunction := dynamicSecretTemplateFunction(accessToken, dynamicSecretLeaseManager, templateId, currentEtag) funcs := template.FuncMap{ "secret": secretFunction, "dynamic_secret": dynamicSecretFunction, @@ -474,9 +853,10 @@ func ProcessBase64Template(templateId int, encodedTemplate string, data interfac return &buf, nil } -func ProcessLiteralTemplate(templateId int, templateString string, data interface{}, accessToken string, existingEtag string, currentEtag *string, dynamicSecretLeaser *DynamicSecretLeaseManager) (*bytes.Buffer, error) { - secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) // TODO: Fix this - dynamicSecretFunction := dynamicSecretTemplateFunction(accessToken, dynamicSecretLeaser, templateId, currentEtag) +func ProcessLiteralTemplate(templateId int, templateString string, data interface{}, accessToken string, existingEtag string, currentEtag *string, dynamicSecretLeaseManager *DynamicSecretLeaseManager) (*bytes.Buffer, error) { + + secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) + dynamicSecretFunction := dynamicSecretTemplateFunction(accessToken, dynamicSecretLeaseManager, templateId, currentEtag) funcs := template.FuncMap{ "secret": secretFunction, "dynamic_secret": dynamicSecretFunction, @@ -507,12 +887,13 @@ type AgentManager struct { filePaths []Sink // Store file paths if needed templates []TemplateWithID dynamicSecretLeases *DynamicSecretLeaseManager - - authConfigBytes []byte - authStrategy util.AuthStrategyType + cacheManager *CacheManager + authConfigBytes []byte + authStrategy util.AuthStrategyType newAccessTokenNotificationChan chan bool cachedUniversalAuthClientSecret string + templateFirstRenderOnce map[int]*sync.Once // Track first render per template exitAfterAuth bool revokeCredentialsOnShutdown bool @@ -540,8 +921,10 @@ func NewAgentManager(options NewAgentMangerOptions) *AgentManager { } templates := make([]TemplateWithID, len(options.Templates)) + templateFirstRenderOnce := make(map[int]*sync.Once) for i, template := range options.Templates { templates[i] = TemplateWithID{ID: i + 1, Template: template} + templateFirstRenderOnce[i+1] = &sync.Once{} } return &AgentManager{ @@ -554,6 +937,7 @@ func NewAgentManager(options NewAgentMangerOptions) *AgentManager { newAccessTokenNotificationChan: options.NewAccessTokenNotificationChan, exitAfterAuth: options.ExitAfterAuth, revokeCredentialsOnShutdown: options.RevokeCredentialsOnShutdown, + templateFirstRenderOnce: templateFirstRenderOnce, infisicalClient: infisicalSdk.NewInfisicalClient(context.Background(), infisicalSdk.Config{ SiteUrl: config.INFISICAL_URL, @@ -772,6 +1156,35 @@ func (tm *AgentManager) FetchNewAccessToken() error { return nil } +func revokeDynamicSecretLease(accessToken, projectSlug, environment, secretPath, leaseID string) error { + customHeaders, err := util.GetInfisicalCustomHeadersMap() + if err != nil { + return fmt.Errorf("unable to get custom headers: %v", err) + } + + temporaryInfisicalClient := infisicalSdk.NewInfisicalClient(context.Background(), infisicalSdk.Config{ + SiteUrl: config.INFISICAL_URL, + UserAgent: api.USER_AGENT, + AutoTokenRefresh: false, + CustomHeaders: customHeaders, + }) + + temporaryInfisicalClient.Auth().SetAccessToken(accessToken) + + _, err = temporaryInfisicalClient.DynamicSecrets().Leases().DeleteById(infisicalSdk.DeleteDynamicSecretLeaseOptions{ + LeaseId: leaseID, + ProjectSlug: projectSlug, + SecretPath: secretPath, + EnvironmentSlug: environment, + }) + if err != nil { + return fmt.Errorf("unable to revoke dynamic secret lease: %v", err) + } + + return nil + +} + func (tm *AgentManager) RevokeCredentials() error { var token string @@ -795,21 +1208,7 @@ func (tm *AgentManager) RevokeCredentials() error { for _, lease := range dynamicSecretLeases { - temporaryInfisicalClient := infisicalSdk.NewInfisicalClient(context.Background(), infisicalSdk.Config{ - SiteUrl: config.INFISICAL_URL, - UserAgent: api.USER_AGENT, - AutoTokenRefresh: false, - CustomHeaders: customHeaders, - }) - - temporaryInfisicalClient.Auth().SetAccessToken(token) - - _, err = temporaryInfisicalClient.DynamicSecrets().Leases().DeleteById(infisicalSdk.DeleteDynamicSecretLeaseOptions{ - LeaseId: lease.LeaseID, - ProjectSlug: lease.ProjectSlug, - SecretPath: lease.SecretPath, - EnvironmentSlug: lease.Environment, - }) + err = revokeDynamicSecretLease(token, lease.ProjectSlug, lease.Environment, lease.SecretPath, lease.LeaseID) if err != nil { @@ -826,12 +1225,7 @@ func (tm *AgentManager) RevokeCredentials() error { // find the template that this lease is associated with templateIndex := slices.IndexFunc(tm.templates, func(t TemplateWithID) bool { - for _, templateID := range lease.TemplateIDs { - if t.ID == templateID { - return true - } - } - return false + return t.ID == lease.TemplateID }) if templateIndex != -1 { @@ -959,14 +1353,16 @@ func (tm *AgentManager) ManageTokenLifecycle() { isSavedTokenValid := false token := tm.FetchTokenFromFiles() if token != "" { - log.Info().Msg("found existing token in file, attempting to refresh...") + + log.Info().Msg("found existing token in cache, attempting to refresh...") err := tm.RefreshAccessToken(token) isSavedTokenValid = err == nil + if isSavedTokenValid { - log.Info().Msg("token refreshed successfully from saved file") + log.Info().Msg("token refreshed successfully from saved cache") tm.accessTokenFetchedTime = time.Now() } else { - log.Error().Msg("unable to refresh token from saved file") + log.Error().Msg("unable to refresh token from saved cache") } } @@ -1006,11 +1402,6 @@ func (tm *AgentManager) ManageTokenLifecycle() { } } - if tm.exitAfterAuth { - time.Sleep(25 * time.Second) - os.Exit(0) - } - if accessTokenRefreshedTime.IsZero() { accessTokenRefreshedTime = tm.accessTokenFetchedTime } else { @@ -1035,6 +1426,7 @@ func (tm *AgentManager) ManageTokenLifecycle() { func (tm *AgentManager) WriteTokenToFiles() { token := tm.GetToken() + for _, sinkFile := range tm.filePaths { if sinkFile.Type == "file" { err := ioutil.WriteFile(sinkFile.Config.Path, []byte(token), 0644) @@ -1068,15 +1460,19 @@ func (tm *AgentManager) FetchTokenFromFiles() string { return "" } -func (tm *AgentManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template) { +func (tm *AgentManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template, templateId int) { if err := WriteBytesToFile(bytes, template.DestinationPath); err != nil { log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) return } - log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", template.SourcePath, template.DestinationPath) + if template.SourcePath != "" { + log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s [template-id=%d]", template.SourcePath, template.DestinationPath, templateId) + } else { + log.Info().Msgf("template engine: secret template has been rendered and saved to path %s [template-id=%d]", template.DestinationPath, templateId) + } } -func (tm *AgentManager) MonitorSecretChanges(secretTemplate Template, templateId int, sigChan chan os.Signal) { +func (tm *AgentManager) MonitorSecretChanges(secretTemplate Template, templateId int, sigChan chan os.Signal, monitoringChan chan bool) { pollingInterval := time.Duration(5 * time.Minute) @@ -1125,17 +1521,30 @@ func (tm *AgentManager) MonitorSecretChanges(secretTemplate Template, templateId } if err != nil { - log.Error().Msgf("unable to process template because %v", err) + log.Error().Msgf("unable to process template because %v [template-id=%d]", err, templateId) // case: if exit-after-auth is true, it should exit the agent once an error on secret fetching occurs with the appropriate exit code (1) // previous behavior would exit after 25 sec with status code 0, even if this step errors if tm.exitAfterAuth { os.Exit(1) } + + // if polling interval is less than 1 minute, we sleep for the polling interval, otherwise we sleep for 1 minute + + sleepDuration := 1 * time.Minute + + if pollingInterval < sleepDuration { + sleepDuration = pollingInterval + } + + log.Info().Msgf("template engine: retrying in %s [template-id=%d]", sleepDuration.String(), templateId) + time.Sleep(sleepDuration) + continue + } else { if (existingEtag != currentEtag) || firstRun { - tm.WriteTemplateToFile(processedTemplate, &secretTemplate) + tm.WriteTemplateToFile(processedTemplate, &secretTemplate, templateId) existingEtag = currentEtag @@ -1150,6 +1559,10 @@ func (tm *AgentManager) MonitorSecretChanges(secretTemplate Template, templateId } if firstRun { firstRun = false + // Signal that this template has completed its first render + tm.templateFirstRenderOnce[templateId].Do(func() { + monitoringChan <- true + }) } } } @@ -1157,11 +1570,12 @@ func (tm *AgentManager) MonitorSecretChanges(secretTemplate Template, templateId // now the idea is we pick the next sleep time in which the one shorter out of // - polling time // - first lease that's gonna get expired in the template - firstLeaseExpiry, isValid := tm.dynamicSecretLeases.GetFirstExpiringLeaseTime(templateId) + firstLeaseExpiry, isValid := tm.dynamicSecretLeases.GetFirstExpiringLeaseTime() var waitTime = pollingInterval if isValid && firstLeaseExpiry.Sub(time.Now()) < pollingInterval { waitTime = firstLeaseExpiry.Sub(time.Now()) } + time.Sleep(waitTime) } else { // It fails to get the access token. So we will re-try in 3 seconds. We do this because if we don't, the user will have to wait for the next polling interval to get the first secret render. @@ -1200,7 +1614,7 @@ var agentCmd = &cobra.Command{ log.Error().Msgf("Unable to locate %s. The provided agent config file path is either missing or incorrect", configPath) return } - } + } // pgrep -f "dev-agent" agentConfigInBytes = data } @@ -1231,7 +1645,10 @@ var agentCmd = &cobra.Command{ util.PrintErrorMessageAndExit(fmt.Sprintf("The auth method '%s' is not supported.", agentConfig.Auth.Type)) } + ctx, cancel := context.WithCancel(context.Background()) + tokenRefreshNotifier := make(chan bool) + monitoringChan := make(chan bool, len(agentConfig.Templates)) sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) @@ -1240,6 +1657,7 @@ var agentCmd = &cobra.Command{ configBytes, err := yaml.Marshal(agentConfig.Auth.Config) if err != nil { log.Error().Msgf("unable to marshal auth config because %v", err) + cancel() return } @@ -1253,13 +1671,63 @@ var agentCmd = &cobra.Command{ RevokeCredentialsOnShutdown: agentConfig.Infisical.RevokeCredentialsOnShutdown, }) - tm.dynamicSecretLeases = NewDynamicSecretLeaseManager(sigChan) + tm.cacheManager, err = NewCacheManager(ctx, &agentConfig.Cache) + if err != nil { + log.Error().Msgf("unable to setup cache manager: %v", err) + cancel() + return + } + tm.dynamicSecretLeases = NewDynamicSecretLeaseManager(sigChan, tm.cacheManager) + + // start a http server that returns a json object of the whole cache + if util.IsDevelopmentMode() && tm.cacheManager != nil && tm.cacheManager.IsEnabled { + + go func() { + http.HandleFunc("/cache", func(w http.ResponseWriter, r *http.Request) { + + all, err := tm.cacheManager.cacheStorage.GetAll() + if err != nil { + log.Error().Msgf("unable to get all cache: %v", err) + json.NewEncoder(w).Encode(map[string]interface{}{"error": err.Error()}) + return + } + + json.NewEncoder(w).Encode(all) + + }) + log.Info().Msg("starting cache http server on port 9000") + http.ListenAndServe(":9000", nil) + }() + } go tm.ManageTokenLifecycle() + var monitoredTemplatesFinished atomic.Int32 + + // when all templates have finished rendering once, we delete the unused leases from the cache. + go func() { + for { + select { + case <-monitoringChan: + monitoredTemplatesFinished.Add(1) + if monitoredTemplatesFinished.Load() == int32(len(tm.templates)) { + if err := tm.dynamicSecretLeases.DeleteUnusedLeasesFromCache(); err != nil { + log.Error().Msgf("[template monitor] failed to delete unused leases from cache: %v", err) + } + + if tm.exitAfterAuth { + os.Exit(0) + } + } + case <-sigChan: + return + } + } + }() + for _, template := range tm.templates { log.Info().Msgf("template engine started for template %v...", template.ID) - go tm.MonitorSecretChanges(template.Template, template.ID, sigChan) + go tm.MonitorSecretChanges(template.Template, template.ID, sigChan, monitoringChan) } for { @@ -1269,6 +1737,7 @@ var agentCmd = &cobra.Command{ case <-sigChan: tm.isShuttingDown = true log.Info().Msg("agent is gracefully shutting down...") + cancel() exitCode := 0 diff --git a/packages/cmd/export.go b/packages/cmd/export.go index 33cd212c..21344ecc 100644 --- a/packages/cmd/export.go +++ b/packages/cmd/export.go @@ -109,7 +109,7 @@ var exportCmd = &cobra.Command{ if templatePath != "" { sigChan := make(chan os.Signal, 1) - dynamicSecretLeases := NewDynamicSecretLeaseManager(sigChan) + dynamicSecretLeases := NewDynamicSecretLeaseManager(sigChan, nil) newEtag := "" accessToken := "" @@ -207,7 +207,7 @@ func resolveOutputPath(outputFile, format string) (string, error) { defaultFilename := getDefaultFilename(format) return filepath.Join(absPath, defaultFilename), nil } - + // Ensure the parent directory exists parentDir := filepath.Dir(absPath) if _, err := os.Stat(parentDir); os.IsNotExist(err) { @@ -216,7 +216,7 @@ func resolveOutputPath(outputFile, format string) (string, error) { return "", fmt.Errorf("failed to create parent directory %s: %w", parentDir, err) } } - + // If no extension provided, add default extension based on format if filepath.Ext(absPath) == "" { ext := getDefaultExtension(format) @@ -357,4 +357,4 @@ func escapeNewLinesIfRequired(env models.SingleEnvironmentVariable) string { } return env.Value -} \ No newline at end of file +} diff --git a/packages/cmd/root.go b/packages/cmd/root.go index b9370ad8..27a85403 100644 --- a/packages/cmd/root.go +++ b/packages/cmd/root.go @@ -38,7 +38,7 @@ func Execute() { func init() { cobra.OnInitialize(initLog) - rootCmd.PersistentFlags().StringP("log-level", "l", "info", "log level (trace, debug, info, warn, error, fatal)") + rootCmd.PersistentFlags().StringP("log-level", "l", "", "log level (trace, debug, info, warn, error, fatal)") rootCmd.PersistentFlags().Bool("telemetry", true, "Infisical collects non-sensitive telemetry data to enhance features and improve user experience. Participation is voluntary") rootCmd.PersistentFlags().StringVar(&config.INFISICAL_URL, "domain", fmt.Sprintf("%s/api", util.INFISICAL_DEFAULT_US_URL), "Point the CLI to your own backend [can also set via environment variable name: INFISICAL_API_URL]") rootCmd.PersistentFlags().Bool("silent", false, "Disable output of tip/info messages. Useful when running in scripts or CI/CD pipelines.") @@ -85,6 +85,15 @@ func initLog() { if err != nil { log.Fatal().Msg(err.Error()) } + + if ll == "" { + ll = os.Getenv("LOG_LEVEL") + + if ll == "" { + ll = "info" + } + } + switch strings.ToLower(ll) { case "trace": zerolog.SetGlobalLevel(zerolog.TraceLevel) diff --git a/packages/gateway-v2/gateway.go b/packages/gateway-v2/gateway.go index 0ae97efb..095870a6 100644 --- a/packages/gateway-v2/gateway.go +++ b/packages/gateway-v2/gateway.go @@ -374,7 +374,7 @@ func (g *Gateway) registerGateway() error { return fmt.Errorf("failed to register gateway: %v", err) } - if util.CLI_VERSION == "devel" && certResp.RelayHost == "host.docker.internal" { + if util.IsDevelopmentMode() && certResp.RelayHost == "host.docker.internal" { certResp.RelayHost = "127.0.0.1" } @@ -510,7 +510,7 @@ func (g *Gateway) createHostKeyCallback() ssh.HostKeyCallback { } // no host cert check when in dev mode - if util.CLI_VERSION == "devel" { + if util.IsDevelopmentMode() { fmt.Println("Gateway running in development mode, skipping host certificate validation") return nil } diff --git a/packages/models/cli.go b/packages/models/cli.go index 785b3827..0b8c117f 100644 --- a/packages/models/cli.go +++ b/packages/models/cli.go @@ -64,6 +64,14 @@ type DynamicSecret struct { Type string `json:"type"` } +type DynamicSecretLeaseWithoutData struct { + Lease struct { + Id string `json:"id"` + ExpireAt time.Time `json:"expireAt"` + } `json:"lease"` + DynamicSecret DynamicSecret `json:"dynamicSecret"` +} + type DynamicSecretLease struct { Lease struct { Id string `json:"id"` diff --git a/packages/util/agent.go b/packages/util/agent.go index 215e4355..6f009106 100644 --- a/packages/util/agent.go +++ b/packages/util/agent.go @@ -23,7 +23,7 @@ func ConvertPollingIntervalToTime(pollingInterval string) (time.Duration, error) switch unit { case "s": - if number < 60 { + if number < 60 && !IsDevelopmentMode() { return 0, fmt.Errorf("polling interval must be at least 60 seconds") } return time.Duration(number) * time.Second, nil diff --git a/packages/util/cache/cache-storage.go b/packages/util/cache/cache-storage.go new file mode 100644 index 00000000..1dee11f9 --- /dev/null +++ b/packages/util/cache/cache-storage.go @@ -0,0 +1,254 @@ +package cache + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "reflect" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/rs/zerolog/log" +) + +type EncryptedStorage struct { + db *badger.DB + key [32]byte +} + +type EncryptedStorageOptions struct { + // Only required if InMemory is false + DBPath string + // If InMemory is true, the database will be stored in memory and will never be persisted on disk + InMemory bool + + // Only required if InMemory is false + EncryptionKey [32]byte +} + +func NewEncryptedStorage(opts EncryptedStorageOptions) (*EncryptedStorage, error) { + + var badgerOptions badger.Options + + if opts.InMemory { + + if opts.DBPath != "" { + return nil, fmt.Errorf("DBPath must be empty if InMemory is true") + } + + badgerOptions = badger.DefaultOptions("").WithInMemory(true).WithLogger(nil) + } else { + if opts.DBPath == "" { + return nil, fmt.Errorf("DBPath must be set if InMemory is false") + } + + badgerOptions = badger.DefaultOptions(opts.DBPath).WithLogger(nil) + } + + db, err := badger.Open(badgerOptions) + if err != nil { + return nil, fmt.Errorf("failed to open badger: %w", err) + } + + return &EncryptedStorage{ + db: db, + key: opts.EncryptionKey, + }, nil +} + +func (s *EncryptedStorage) GetAll() (map[string]interface{}, error) { + result := make(map[string]interface{}) + + err := s.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = 10 + it := txn.NewIterator(opts) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + key := string(item.Key()) + + var encrypted []byte + encrypted, err := item.ValueCopy(nil) + if err != nil { + return fmt.Errorf("failed to copy value for key %s: %w", key, err) + } + + decrypted, err := s.decrypt(encrypted) + if err != nil { + return fmt.Errorf("failed to decrypt value for key %s: %w", key, err) + } + + var value interface{} + if err := json.Unmarshal(decrypted, &value); err != nil { + return fmt.Errorf("failed to unmarshal value for key %s: %w", key, err) + } + + result[key] = value + } + return nil + }) + + if err != nil { + return nil, err + } + + return result, nil +} + +func (s *EncryptedStorage) Set(key string, value interface{}) error { + data, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("marshal failed: %w", err) + } + + encrypted, err := s.encrypt(data) + if err != nil { + return fmt.Errorf("encryption failed: %w", err) + } + + return s.db.Update(func(txn *badger.Txn) error { + return txn.Set([]byte(key), encrypted) + }) +} + +// Same as Set but with a TTL. Currently unused, but could be useful for proxying functionality in the future. +func (s *EncryptedStorage) SetWithTTL(key string, value interface{}, ttl time.Duration) error { + + data, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("marshal failed: %w", err) + } + + encrypted, err := s.encrypt(data) + if err != nil { + return fmt.Errorf("encryption failed: %w", err) + } + + return s.db.Update(func(txn *badger.Txn) error { + entry := badger.NewEntry([]byte(key), encrypted).WithTTL(ttl) + return txn.SetEntry(entry) + }) +} + +func (s *EncryptedStorage) Get(key string, dest interface{}) error { + + var encrypted []byte + + err := s.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { + return err + } + encrypted, err = item.ValueCopy(nil) + return err + }) + + if err != nil { + return fmt.Errorf("get failed: %w", err) + } + + decrypted, err := s.decrypt(encrypted) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + + // if dest is a pointer to a pointer, allocate a new value and unmarshal the decrypted data into it. + // without this it will fail to unmarshal if the destination is a pointer to a pointer + rv := reflect.ValueOf(dest) + if rv.Kind() == reflect.Ptr && rv.Elem().Kind() == reflect.Ptr { + // Allocate new value + newVal := reflect.New(rv.Elem().Type().Elem()) + if err := json.Unmarshal(decrypted, newVal.Interface()); err != nil { + return err + } + rv.Elem().Set(newVal) + return nil + } + + return json.Unmarshal(decrypted, dest) +} + +func (s *EncryptedStorage) Delete(key string) error { + + return s.db.Update(func(txn *badger.Txn) error { + return txn.Delete([]byte(key)) + }) +} + +func (s *EncryptedStorage) Close() error { + return s.db.Close() +} + +func (s *EncryptedStorage) ManualGarbageCollection() error { + return s.db.RunValueLogGC(0.5) // 50% of the value log will be garbage collected +} + +func (s *EncryptedStorage) StartPeriodicGarbageCollection(context context.Context) { + + // always run the garbage collection once on call + err := s.ManualGarbageCollection() + if err != nil && err != badger.ErrNoRewrite { + log.Warn().Msgf("failed to run caching garbage collection: %v", err) + } + + ticker := time.NewTicker(15 * time.Minute) + go func() { + + for { + select { + case <-context.Done(): + return + case <-ticker.C: + err := s.db.RunValueLogGC(0.5) // 50% of the value log will be garbage collected + if err != nil && err != badger.ErrNoRewrite { + log.Warn().Msgf("failed to run caching garbage collection: %v", err) + } + } + } + }() +} + +func (s *EncryptedStorage) encrypt(plaintext []byte) ([]byte, error) { + block, err := aes.NewCipher(s.key[:]) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + return gcm.Seal(nonce, nonce, plaintext, nil), nil +} + +func (s *EncryptedStorage) decrypt(ciphertext []byte) ([]byte, error) { + block, err := aes.NewCipher(s.key[:]) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + nonceSize := gcm.NonceSize() + if len(ciphertext) < nonceSize { + return nil, fmt.Errorf("ciphertext too short") + } + + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + return gcm.Open(nil, nonce, ciphertext, nil) +} diff --git a/packages/util/helper.go b/packages/util/helper.go index 3969dea9..208a8de1 100644 --- a/packages/util/helper.go +++ b/packages/util/helper.go @@ -358,3 +358,7 @@ func GenerateETagFromSecrets(secrets []models.SingleEnvironmentVariable) string hash := sha256.Sum256(content) return fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:])) } + +func IsDevelopmentMode() bool { + return CLI_VERSION == "devel" +} diff --git a/packages/util/secrets.go b/packages/util/secrets.go index ab592e53..0dd69c05 100644 --- a/packages/util/secrets.go +++ b/packages/util/secrets.go @@ -162,6 +162,32 @@ func GetSinglePlainTextSecretByNameV3(accessToken string, workspaceId string, en return formattedSecrets, rawSecret.ETag, nil } +func GetDynamicSecretLease(accessToken string, projectSlug string, environmentSlug string, secretsPath string, leaseID string) (models.DynamicSecretLeaseWithoutData, error) { + httpClient, err := GetRestyClientWithCustomHeaders() + if err != nil { + return models.DynamicSecretLeaseWithoutData{}, err + } + + httpClient.SetAuthToken(accessToken). + SetHeader("Accept", "application/json") + + dynamicSecret, err := api.CallGetDynamicSecretLeaseV1(httpClient, api.GetDynamicSecretLeaseV1Request{ + LeaseID: leaseID, + Environment: environmentSlug, + ProjectSlug: projectSlug, + SecretPath: secretsPath, + }) + + if err != nil { + return models.DynamicSecretLeaseWithoutData{}, err + } + + return models.DynamicSecretLeaseWithoutData{ + Lease: dynamicSecret.Lease, + DynamicSecret: dynamicSecret.DynamicSecret, + }, nil +} + func CreateDynamicSecretLease(accessToken string, projectSlug string, environmentSlug string, secretsPath string, dynamicSecretName string, ttl string) (models.DynamicSecretLease, error) { httpClient, err := GetRestyClientWithCustomHeaders() if err != nil {