From f025a25cf26bb6e0909c23c08c3d8c73c46d9bea Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 22 Jan 2025 16:55:55 -0800 Subject: [PATCH 01/63] start work on events --- packages/cdevents/endpoint.go | 91 ++++++++++++++++++++++++++++++++ packages/cdevents/go.mod | 3 ++ packages/cloudevents/endpoint.go | 73 +++++++++++++++++++++++++ packages/cloudevents/go.mod | 3 ++ 4 files changed, 170 insertions(+) create mode 100644 packages/cdevents/endpoint.go create mode 100644 packages/cdevents/go.mod create mode 100644 packages/cloudevents/endpoint.go create mode 100644 packages/cloudevents/go.mod diff --git a/packages/cdevents/endpoint.go b/packages/cdevents/endpoint.go new file mode 100644 index 000000000..1805c4fa1 --- /dev/null +++ b/packages/cdevents/endpoint.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + cdevents "github.com/cdevents/sdk-go/pkg/api" + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + _ "github.com/lib/pq" + "os" +) + +const ( + port = 5432 + user = "postgres" + dbname = "postgres" +) + +var logger = log.NewLogfmtLogger(os.Stdout) + +func CheckError(err error) { + if err != nil { + level.Error(logger).Log("error", err) + panic(err) + } +} + +func main() { + + // The default client is HTTP. + c, err := cloudevents.NewClientHTTP() + if err != nil { + level.Error(logger).Log("failed to create client, %v", err) + return + } + level.Error(logger).Log(c.StartReceiver(context.Background(), receiveCDevent)) +} + +func receiveCDevent(event cloudevents.Event) { + + eventType, err := cdevents.ParseType(event.Type()) + if err != nil { + level.Debug(logger).Log("this is not a valid CDEvent", eventType) + return + } + + // do something with event. + //level.Debug(logger).Log("ce event", fmt.Sprintf("%s", event)) + + level.Debug(logger).Log("json ce event data", event.Data()) + + cdEvent, err := cdevents.NewFromJsonBytes(event.Data()) + if err != nil { + level.Debug(logger).Log("Failed to get CDevent from event.data() -> err: ", err) + level.Debug(logger).Log("Failed to get CDevent from event.data()", cdEvent) + return + } + + level.Debug(logger).Log("cdevent ", cdEvent) + // connection string + psqlconn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", os.Getenv("POSTGRESQL_HOST"), port, user, os.Getenv("POSTGRESQL_PASS"), dbname) + + // open database + db, err := sql.Open("postgres", psqlconn) + CheckError(err) + + // close database + defer db.Close() + + // check db + err = db.Ping() + CheckError(err) + + level.Info(logger).Log("DB", "Connected!") + + jsonCDEvent, err := json.Marshal(cdEvent) + + if err != nil { + level.Debug(logger).Log("Failed to marshal CDevent to json ", err) + return + } + + // insert + insertStmt := `insert into "cdevents_raw"("cd_source", "cd_id", "cd_timestamp", "cd_type", "cd_subject_id", "cd_subject_source", "content") values($1, $2, $3, $4, $5 , $6, $7)` + _, e := db.Exec(insertStmt, cdEvent.GetSource(), cdEvent.GetId(), cdEvent.GetTimestamp(), cdEvent.GetType(), cdEvent.GetSubjectId(), cdEvent.GetSubjectSource(), jsonCDEvent) + CheckError(e) + +} diff --git a/packages/cdevents/go.mod b/packages/cdevents/go.mod new file mode 100644 index 000000000..f257c35ea --- /dev/null +++ b/packages/cdevents/go.mod @@ -0,0 +1,3 @@ +module github.com/aws-samples/appmod-blueprints/packages/cdevents + +go 1.23.4 diff --git a/packages/cloudevents/endpoint.go b/packages/cloudevents/endpoint.go new file mode 100644 index 000000000..66a55bdcb --- /dev/null +++ b/packages/cloudevents/endpoint.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "os" + + cloudevents "github.com/cloudevents/sdk-go/v2" + _ "github.com/lib/pq" +) + +const ( + port = 5432 + user = "postgres" + dbname = "postgres" +) + +var logger = log.NewLogfmtLogger(os.Stdout) + +func CheckError(err error) { + if err != nil { + level.Error(logger).Log("error", err) + panic(err) + } +} + +// cloudevents_raw table `CREATE TABLE IF NOT EXISTS cloudevents_raw ( event_id serial NOT NULL PRIMARY KEY, content json NOT NULL, event_timestamp TIMESTAMP NOT NULL);` +func main() { + + // The default client is HTTP. + c, err := cloudevents.NewClientHTTP() + if err != nil { + level.Error(logger).Log("failed to create client, %v", err) + } + level.Error(logger).Log(c.StartReceiver(context.Background(), receiveCloudEvent)) +} + +func receiveCloudEvent(event cloudevents.Event) { + + // do something with event. + level.Debug(logger).Log("event", event) + jsonEvent, err := json.Marshal(event) + CheckError(err) + + // connection string + psqlconn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", os.Getenv("POSTGRESQL_HOST"), port, user, os.Getenv("POSTGRESQL_PASS"), dbname) + + // open database + db, err := sql.Open("postgres", psqlconn) + CheckError(err) + + // close database + defer db.Close() + + // check db + err = db.Ping() + CheckError(err) + + level.Info(logger).Log("DB", "Connected!") + + // insert + insertStmt := `insert into "cloudevents_raw"("content", "event_timestamp") values($1, current_timestamp)` + _, e := db.Exec(insertStmt, string(jsonEvent)) + if e != nil { + level.Error(logger).Log("Inserting failed for event ", e) + } + CheckError(e) + +} diff --git a/packages/cloudevents/go.mod b/packages/cloudevents/go.mod new file mode 100644 index 000000000..1101a7e1c --- /dev/null +++ b/packages/cloudevents/go.mod @@ -0,0 +1,3 @@ +module github.com/aws-samples/appmod-blueprints/packages/cloudevents + +go 1.23.4 From 38e6abd065962aac01ad396dbbc0d0d557fe1cbf Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 20 Feb 2025 11:33:12 -0800 Subject: [PATCH 02/63] old cloudevents stuff --- packages/cdevents/endpoint.go | 2 +- packages/cdevents/go.mod | 28 ++++++++++ packages/cdevents/go.sum | 101 ++++++++++++++++++++++++++++++++++ packages/cloudevents/go.mod | 17 ++++++ packages/cloudevents/go.sum | 44 +++++++++++++++ 5 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 packages/cdevents/go.sum create mode 100644 packages/cloudevents/go.sum diff --git a/packages/cdevents/endpoint.go b/packages/cdevents/endpoint.go index 1805c4fa1..73c4fecb7 100644 --- a/packages/cdevents/endpoint.go +++ b/packages/cdevents/endpoint.go @@ -5,7 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" - cdevents "github.com/cdevents/sdk-go/pkg/api" + cdevents "github.com/cdevents/sdk-go/pkg/api/v04" cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" diff --git a/packages/cdevents/go.mod b/packages/cdevents/go.mod index f257c35ea..396f95317 100644 --- a/packages/cdevents/go.mod +++ b/packages/cdevents/go.mod @@ -1,3 +1,31 @@ module github.com/aws-samples/appmod-blueprints/packages/cdevents go 1.23.4 + +require ( + github.com/cdevents/sdk-go v0.4.1 + github.com/cloudevents/sdk-go/v2 v2.15.2 + github.com/go-kit/log v0.2.1 + github.com/lib/pq v1.10.9 +) + +require ( + github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.11.1 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/package-url/packageurl-go v0.1.1 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.17.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/packages/cdevents/go.sum b/packages/cdevents/go.sum new file mode 100644 index 000000000..b5076149b --- /dev/null +++ b/packages/cdevents/go.sum @@ -0,0 +1,101 @@ +github.com/cdevents/sdk-go v0.4.1 h1:Cr/iH/I51Z+slxKRx9AV7stn6hr2pjRHQ5wpPJhRLTU= +github.com/cdevents/sdk-go v0.4.1/go.mod h1:3IhWLoY4vsyUEzv7XJbyr0BRQ0KPgvNx+wiD2hQGFNU= +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/package-url/packageurl-go v0.1.1 h1:KTRE0bK3sKbFKAk3yy63DpeskU7Cvs/x/Da5l+RtzyU= +github.com/package-url/packageurl-go v0.1.1/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudevents/go.mod b/packages/cloudevents/go.mod index 1101a7e1c..6c512d275 100644 --- a/packages/cloudevents/go.mod +++ b/packages/cloudevents/go.mod @@ -1,3 +1,20 @@ module github.com/aws-samples/appmod-blueprints/packages/cloudevents go 1.23.4 + +require ( + github.com/cloudevents/sdk-go/v2 v2.15.2 + github.com/go-kit/log v0.2.1 + github.com/lib/pq v1.10.9 +) + +require ( + github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/json-iterator/go v1.1.10 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + go.uber.org/atomic v1.4.0 // indirect + go.uber.org/multierr v1.1.0 // indirect + go.uber.org/zap v1.10.0 // indirect +) diff --git a/packages/cloudevents/go.sum b/packages/cloudevents/go.sum new file mode 100644 index 000000000..b68067199 --- /dev/null +++ b/packages/cloudevents/go.sum @@ -0,0 +1,44 @@ +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 6e1140dbdfbc51a2024e5e574d1f7aa20ccebd4e Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 4 Mar 2025 17:20:29 -0800 Subject: [PATCH 03/63] remove cloud events and switch to devlake --- .../argocd/dev/appproject-modern-engg.yaml | 2 + packages/cdevents/endpoint.go | 91 ---- packages/cdevents/go.mod | 31 -- packages/cdevents/go.sum | 101 ----- packages/cloudevents/endpoint.go | 73 --- packages/cloudevents/go.mod | 20 - packages/cloudevents/go.sum | 44 -- packages/devlake/dev/external-secrets.yaml | 19 + packages/devlake/dev/kustomization.yaml | 3 + .../dev/patches/deployment-devlake-ui.yaml | 11 + .../dev/patches/deployment-devlake.yaml | 11 + packages/devlake/dev/values.yaml | 420 ++++++++++++++++++ .../infra/terraform/mgmt/terraform/devlake.tf | 140 ++++++ .../templates/argocd-apps/devlake.yaml | 29 ++ .../manifests/devlake-secret-store.yaml | 14 + .../templates/manifests/ingress-devlake.yaml | 0 16 files changed, 649 insertions(+), 360 deletions(-) delete mode 100644 packages/cdevents/endpoint.go delete mode 100644 packages/cdevents/go.mod delete mode 100644 packages/cdevents/go.sum delete mode 100644 packages/cloudevents/endpoint.go delete mode 100644 packages/cloudevents/go.mod delete mode 100644 packages/cloudevents/go.sum create mode 100644 packages/devlake/dev/external-secrets.yaml create mode 100644 packages/devlake/dev/kustomization.yaml create mode 100644 packages/devlake/dev/patches/deployment-devlake-ui.yaml create mode 100644 packages/devlake/dev/patches/deployment-devlake.yaml create mode 100644 packages/devlake/dev/values.yaml create mode 100644 platform/infra/terraform/mgmt/terraform/devlake.tf create mode 100644 platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml create mode 100644 platform/infra/terraform/mgmt/terraform/templates/manifests/devlake-secret-store.yaml create mode 100644 platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml diff --git a/packages/argocd/dev/appproject-modern-engg.yaml b/packages/argocd/dev/appproject-modern-engg.yaml index 321048e3b..cedfc2892 100644 --- a/packages/argocd/dev/appproject-modern-engg.yaml +++ b/packages/argocd/dev/appproject-modern-engg.yaml @@ -22,6 +22,8 @@ spec: server: https://kubernetes.default.svc - namespace: dapr-system server: https://kubernetes.default.svc + - namespace: devlake + server: https://kubernetes.default.svc - namespace: ingress-nginx server: https://kubernetes.default.svc - namespace: kyverno diff --git a/packages/cdevents/endpoint.go b/packages/cdevents/endpoint.go deleted file mode 100644 index 73c4fecb7..000000000 --- a/packages/cdevents/endpoint.go +++ /dev/null @@ -1,91 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - cdevents "github.com/cdevents/sdk-go/pkg/api/v04" - cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - _ "github.com/lib/pq" - "os" -) - -const ( - port = 5432 - user = "postgres" - dbname = "postgres" -) - -var logger = log.NewLogfmtLogger(os.Stdout) - -func CheckError(err error) { - if err != nil { - level.Error(logger).Log("error", err) - panic(err) - } -} - -func main() { - - // The default client is HTTP. - c, err := cloudevents.NewClientHTTP() - if err != nil { - level.Error(logger).Log("failed to create client, %v", err) - return - } - level.Error(logger).Log(c.StartReceiver(context.Background(), receiveCDevent)) -} - -func receiveCDevent(event cloudevents.Event) { - - eventType, err := cdevents.ParseType(event.Type()) - if err != nil { - level.Debug(logger).Log("this is not a valid CDEvent", eventType) - return - } - - // do something with event. - //level.Debug(logger).Log("ce event", fmt.Sprintf("%s", event)) - - level.Debug(logger).Log("json ce event data", event.Data()) - - cdEvent, err := cdevents.NewFromJsonBytes(event.Data()) - if err != nil { - level.Debug(logger).Log("Failed to get CDevent from event.data() -> err: ", err) - level.Debug(logger).Log("Failed to get CDevent from event.data()", cdEvent) - return - } - - level.Debug(logger).Log("cdevent ", cdEvent) - // connection string - psqlconn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", os.Getenv("POSTGRESQL_HOST"), port, user, os.Getenv("POSTGRESQL_PASS"), dbname) - - // open database - db, err := sql.Open("postgres", psqlconn) - CheckError(err) - - // close database - defer db.Close() - - // check db - err = db.Ping() - CheckError(err) - - level.Info(logger).Log("DB", "Connected!") - - jsonCDEvent, err := json.Marshal(cdEvent) - - if err != nil { - level.Debug(logger).Log("Failed to marshal CDevent to json ", err) - return - } - - // insert - insertStmt := `insert into "cdevents_raw"("cd_source", "cd_id", "cd_timestamp", "cd_type", "cd_subject_id", "cd_subject_source", "content") values($1, $2, $3, $4, $5 , $6, $7)` - _, e := db.Exec(insertStmt, cdEvent.GetSource(), cdEvent.GetId(), cdEvent.GetTimestamp(), cdEvent.GetType(), cdEvent.GetSubjectId(), cdEvent.GetSubjectSource(), jsonCDEvent) - CheckError(e) - -} diff --git a/packages/cdevents/go.mod b/packages/cdevents/go.mod deleted file mode 100644 index 396f95317..000000000 --- a/packages/cdevents/go.mod +++ /dev/null @@ -1,31 +0,0 @@ -module github.com/aws-samples/appmod-blueprints/packages/cdevents - -go 1.23.4 - -require ( - github.com/cdevents/sdk-go v0.4.1 - github.com/cloudevents/sdk-go/v2 v2.15.2 - github.com/go-kit/log v0.2.1 - github.com/lib/pq v1.10.9 -) - -require ( - github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator/v10 v10.11.1 // indirect - github.com/google/uuid v1.1.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/leodido/go-urn v1.2.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/package-url/packageurl-go v0.1.1 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.17.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/mod v0.18.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.14.0 // indirect -) diff --git a/packages/cdevents/go.sum b/packages/cdevents/go.sum deleted file mode 100644 index b5076149b..000000000 --- a/packages/cdevents/go.sum +++ /dev/null @@ -1,101 +0,0 @@ -github.com/cdevents/sdk-go v0.4.1 h1:Cr/iH/I51Z+slxKRx9AV7stn6hr2pjRHQ5wpPJhRLTU= -github.com/cdevents/sdk-go v0.4.1/go.mod h1:3IhWLoY4vsyUEzv7XJbyr0BRQ0KPgvNx+wiD2hQGFNU= -github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= -github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= -github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/package-url/packageurl-go v0.1.1 h1:KTRE0bK3sKbFKAk3yy63DpeskU7Cvs/x/Da5l+RtzyU= -github.com/package-url/packageurl-go v0.1.1/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudevents/endpoint.go b/packages/cloudevents/endpoint.go deleted file mode 100644 index 66a55bdcb..000000000 --- a/packages/cloudevents/endpoint.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "os" - - cloudevents "github.com/cloudevents/sdk-go/v2" - _ "github.com/lib/pq" -) - -const ( - port = 5432 - user = "postgres" - dbname = "postgres" -) - -var logger = log.NewLogfmtLogger(os.Stdout) - -func CheckError(err error) { - if err != nil { - level.Error(logger).Log("error", err) - panic(err) - } -} - -// cloudevents_raw table `CREATE TABLE IF NOT EXISTS cloudevents_raw ( event_id serial NOT NULL PRIMARY KEY, content json NOT NULL, event_timestamp TIMESTAMP NOT NULL);` -func main() { - - // The default client is HTTP. - c, err := cloudevents.NewClientHTTP() - if err != nil { - level.Error(logger).Log("failed to create client, %v", err) - } - level.Error(logger).Log(c.StartReceiver(context.Background(), receiveCloudEvent)) -} - -func receiveCloudEvent(event cloudevents.Event) { - - // do something with event. - level.Debug(logger).Log("event", event) - jsonEvent, err := json.Marshal(event) - CheckError(err) - - // connection string - psqlconn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", os.Getenv("POSTGRESQL_HOST"), port, user, os.Getenv("POSTGRESQL_PASS"), dbname) - - // open database - db, err := sql.Open("postgres", psqlconn) - CheckError(err) - - // close database - defer db.Close() - - // check db - err = db.Ping() - CheckError(err) - - level.Info(logger).Log("DB", "Connected!") - - // insert - insertStmt := `insert into "cloudevents_raw"("content", "event_timestamp") values($1, current_timestamp)` - _, e := db.Exec(insertStmt, string(jsonEvent)) - if e != nil { - level.Error(logger).Log("Inserting failed for event ", e) - } - CheckError(e) - -} diff --git a/packages/cloudevents/go.mod b/packages/cloudevents/go.mod deleted file mode 100644 index 6c512d275..000000000 --- a/packages/cloudevents/go.mod +++ /dev/null @@ -1,20 +0,0 @@ -module github.com/aws-samples/appmod-blueprints/packages/cloudevents - -go 1.23.4 - -require ( - github.com/cloudevents/sdk-go/v2 v2.15.2 - github.com/go-kit/log v0.2.1 - github.com/lib/pq v1.10.9 -) - -require ( - github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/google/uuid v1.1.1 // indirect - github.com/json-iterator/go v1.1.10 // indirect - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect - github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect - go.uber.org/atomic v1.4.0 // indirect - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.10.0 // indirect -) diff --git a/packages/cloudevents/go.sum b/packages/cloudevents/go.sum deleted file mode 100644 index b68067199..000000000 --- a/packages/cloudevents/go.sum +++ /dev/null @@ -1,44 +0,0 @@ -github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= -github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/devlake/dev/external-secrets.yaml b/packages/devlake/dev/external-secrets.yaml new file mode 100644 index 000000000..602d0f487 --- /dev/null +++ b/packages/devlake/dev/external-secrets.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: devlake-encryption-secret + namespace: devlake +spec: + refreshInterval: 5m + secretStoreRef: + name: devlake + kind: SecretStore + target: + name: devlake-encryption-secret + creationPolicy: Owner + data: + - secretKey: ENCRYPTION_SECRET + remoteRef: + key: modern-engg/devlake/encryption + property: ENCRYPTION_SECRET diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml new file mode 100644 index 000000000..9feedb128 --- /dev/null +++ b/packages/devlake/dev/kustomization.yaml @@ -0,0 +1,3 @@ +patchesStrategicMerge: + - patches/deployment-devlake-ui + - pathes/deployment-devlake diff --git a/packages/devlake/dev/patches/deployment-devlake-ui.yaml b/packages/devlake/dev/patches/deployment-devlake-ui.yaml new file mode 100644 index 000000000..390e2188f --- /dev/null +++ b/packages/devlake/dev/patches/deployment-devlake-ui.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: devlake-ui + namespace: devlake +spec: + template: + spec: + containers: + - name: devlake + image: ecr-one diff --git a/packages/devlake/dev/patches/deployment-devlake.yaml b/packages/devlake/dev/patches/deployment-devlake.yaml new file mode 100644 index 000000000..e56dde2c3 --- /dev/null +++ b/packages/devlake/dev/patches/deployment-devlake.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: devlake-lake + namespace: devlake +spec: + template: + spec: + containers: + - name: devlake + image: ecr-one diff --git a/packages/devlake/dev/values.yaml b/packages/devlake/dev/values.yaml new file mode 100644 index 000000000..0d4f62b7d --- /dev/null +++ b/packages/devlake/dev/values.yaml @@ -0,0 +1,420 @@ +imageTag: v1.0.2-beta6 + +# image pull secrets +imagePullSecrets: [] + +#the common environments for all pods except grafana, grafana needs to be set in grafana section seperately +commonEnvs: + TZ: "UTC" + +mysql: + # if use external mysql server, please set true + # by default using false, chart will create a single mysql instance + useExternal: false + + # the external mysql server address + externalServer: 127.0.0.1 + + # external mysql port + externalPort: 3306 + + # the username for devlake database + username: merico + + # the password for devlake database + password: merico + + # the database for devlake + database: lake + + # root password for mysql, only used when use_external=false + rootPassword: admin + + # storage for mysql + storage: + # pvc or hostpath + type: pvc + # the storage class for pv, leave empty will using default + class: "" + size: 50Gi + hostPath: /devlake/mysql/data + + # image for mysql + image: + repository: mysql + tag: 8 + pullPolicy: IfNotPresent + + # init containers for mysql if have + initContainers: [] + + # resources config for mysql if have + resources: {} + + # nodeSelector config for mysql if have + nodeSelector: {} + + # tolerations config for mysql if have + tolerations: [] + + # affinity config for mysql if have + affinity: {} + + extraLabels: {} + + securityContext: {} + + containerSecurityContext: {} + + podAnnotations: {} + + service: + type: "ClusterIP" + nodePort: "" + loadBalancerIP: "" + extraLabels: {} + +# pgsql: +# # if use external pgsql server, please set true +# # by default using false, chart will create a single pgsql instance +# useExternal: false + +# # the external pgsql server address +# externalServer: 127.0.0.1 + +# # external pgsql port +# externalPort: 5432 +# # the username for devlake database +# username: merico + +# # the password for devlake database +# password: merico + +# # the database for devlake +# database: lake + +# # storage for pgsql +# storage: +# # the storage class for pv, leave empty will using default +# class: "" +# size: 5Gi + +# # image for pgsql +# image: +# repository: postgres +# tag: 14.5 +# pullPolicy: IfNotPresent + +# # resources config for pgsql if have +# resources: {} + +# # nodeSelector config for pgsql if have +# nodeSelector: {} + +# # tolerations config for pgsql if have +# tolerations: [] + +# # affinity config for pgsql if have +# affinity: {} + +# extraLabels: {} + +# securityContext: {} + +# containerSecurityContext: {} + +# annotations: {} + +# dependency chart values +grafana: + enabled: true + #if grafana enabled is false, then external url should be provided + external: + url: "" + image: + repository: devlake.docker.scarf.sh/apache/devlake-dashboard + tag: v1.0.2-beta6 + deploymentStrategy: + type: Recreate + adminPassword: "" + grafana.ini: + server: + serve_from_subpath: "true" + root_url: "%(protocol)s://%(domain)s/grafana" + #the secret name should be as same as .Values.option.connectionSecretName + envFromSecrets: + - name: "devlake-mysql-auth" + #keep grafana timezone same as other pods, which is set by .Values.commonEnvs.TZ + env: + TZ: "UTC" + persistence: + enabled: true + size: 4Gi + ingressServiceName: "" + ingressServicePort: "" + +lake: + image: + repository: devlake.docker.scarf.sh/apache/devlake + pullPolicy: Always + # defaults to imageTag; if set, lake.image.tag will override imageTag + # tag: + # storage for config + port: 8080 + envs: + API_TIMEOUT: "120s" + API_RETRY: "3" + API_REQUESTS_PER_HOUR: "10000" + PIPELINE_MAX_PARALLEL: "1" + IN_SECURE_SKIP_VERIFY: "false" + LOGGING_DIR: "/app/logs" + # debug, info, warn, error + LOGGING_LEVEL: "info" + JIRA_JQL_AUTO_FULL_REFRESH: "true" + ########################## + # ENABLE_SUBTASKS_BY_DEFAULT: This environment variable is used to enable or disable the execution of subtasks. + # The format is as follows: plugin_name1:subtask_name1:enabled_value,plugin_name2:subtask_name2:enabled_value,plugin_name3:subtask_name3:enabled_value + ########################## + # ENABLE_SUBTASKS_BY_DEFAULT default value as below: + # ENABLE_SUBTASKS_BY_DEFAULT: "jira:collectIssueChangelogs:true,jira:extractIssueChangelogs:true,jira:convertIssueChangelogs:true,tapd:collectBugChangelogs:true,tapd:extractBugChangelogs:true,tapd:convertBugChangelogs:true,zentao:collectBugRepoCommits:true,zentao:extractBugRepoCommits:true,zentao:convertBugRepoCommits:true,zentao:collectStoryRepoCommits:true,zentao:extractStoryRepoCommits:true,zentao:convertStoryRepoCommits:true,zentao:collectTaskRepoCommits:true,zentao:extractTaskRepoCommits:true,zentao:convertTaskRepoCommits:true" + + #extra envs from an existing secret + extraEnvsFromSecret: "" + encryptionSecret: + # The name of secret which contains keys named ENCRYPTION_SECRET + secretName: "" + # if secretName is empty, secret should be set + # you can generate the encryption secret via cmd `openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1` + secret: "" + autoCreateSecret: true + + # If hostNetwork is true, then dnsPolicy is set to ClusterFirstWithHostNet + hostNetwork: false + + resources: {} + + strategy: + type: Recreate + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + extraLabels: {} + + securityContext: {} + + containerSecurityContext: {} + + podAnnotations: {} + + livenessProbe: + httpGet: + path: /ping + port: 8080 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + readinessProbe: + httpGet: + path: /ping + port: 8080 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + deployment: + extraLabels: {} + + service: + extraLabels: {} + + # Additional volumes to include in the Pod. Example: + # + # volumes: + # - name: my-volume + # configMap: my-config-map + volumes: [] + + # Additional volume mounts to include in the Container. Example: + # + # volumeMounts: + # - name: test-volume + # mountPath: /opt/test_folder + # subPath: test_file.yaml + volumeMounts: [] + +ui: + image: + repository: devlake.docker.scarf.sh/apache/devlake-config-ui + pullPolicy: Always + # defaults to imageTag; if set, lake.image.tag will override imageTag + # tag: + resources: {} + + strategy: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + livenessProbe: + httpGet: + path: /health/ + port: 4000 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 15 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + readinessProbe: + httpGet: + path: /health/ + port: 4000 + scheme: HTTP + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + + basicAuth: + enabled: false + user: admin + password: admin + autoCreateSecret: true + secretName: "" + + extraLabels: {} + + podAnnotations: {} + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 101 and gid 1000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: + {} + # fsGroup: 101 + # runAsGroup: 1000 + # runAsNonRoot: true + # runAsUser: 101 + + ## K8s containers' Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - all + + deployment: + extraLabels: {} + + service: + extraLabels: {} + + ## Side Contaainer Configuration + extraContainers: [] +# - name: vault-agent +# image: vault:1.6.2 +# args: +# - agent +# - -config +# - /vault/config/config.hcl +# env: +# - name: VAULT_ADDR +# value: https://vault:8200 +# - name: VAULT_SKIP_VERIFY +# value: "false" +# - name: VAULT_CACERT +# value: /vault/tls/ca.crt + +# alpine image for some init containers +alpine: + image: + repository: alpine + tag: 3.16 + pullPolicy: IfNotPresent + +service: + # service type: NodePort/ClusterIP + type: NodePort + # node port for devlake-ui if NodePort is enabled + uiPort: 32001 + +ingress: + enabled: false + enableHttps: false + # Set to false if you want to use a different ingress controller + useDefaultNginx: false + # ingress class name, example: alb for AWS load balancer controller + className: + # domain name for hosting devlake, must be set if ingress is enabled + hostname: localhost + extraLabels: {} + # annotations required for your ingress controller; see the examples below + # for nginx, use the first two lines of annotations + # for alb (w/ external-dns), use the last 5 (6) lines of annotations + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # + # alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + # alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:xxx:certificate/xxx-xxx-xxx + # alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]' + # alb.ingress.kubernetes.io/scheme: internet-facing + # alb.ingress.kubernetes.io/target-type: ip + # external-dns.alpha.kubernetes.io/hostname: www.example.com + + # url prefix, not works right now, keep "/" + prefix: / + # if using https provides the certificates secret name + tlsSecretName: "" + # ingress http port + httpPort: 80 + # ingress https port + httpsPort: 443 + + extraPaths: [] +# extraPaths: +# - path: /* +# pathType: ImplementationSpecific +# backend: +# service: +# name: ssl-redirect +# port: +# name: use-annotation + +option: + # database type, supported: [mysql] + database: mysql + # the existing k8s secret name of db connection auth. The secret name should be as same as .Values.grafana.envFromSecret + connectionSecretName: "devlake-mysql-auth" + autoCreateSecret: true + +# Define some extra resources to be created +# This section is useful when you need ExternalResource or Secrets, etc. +extraResources: [] + # - apiVersion: v1 + # kind: Secret + # metadata: + # name: example-secret + # labels: {} + # type: Opaque + # stringData: + # username: admin + # password: mypassword diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf new file mode 100644 index 000000000..fcba4a4e9 --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -0,0 +1,140 @@ +resource "aws_iam_policy" "external-secrets-devlake" { + count = local.secret_count + + name_prefix = "modern-engg-external-secrets-devlake-" + description = "For use with External Secrets Controller for Devlake" + policy = jsonencode( + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "secretsmanager:GetResourcePolicy", + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecretVersionIds" + ], + "Resource" : [ + "arn:aws:secretsmanager:${var.region}:${data.aws_caller_identity.current.account_id}:secret:modern-engg/devlake/*" + ] + } + ] + } + ) +} + +module "external_secrets_role_devlake" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.14" + count = local.secret_count + + role_name_prefix = "modern-engg-external-secrets-devlake-" + + oidc_providers = { + main = { + provider_arn = data.aws_iam_openid_connect_provider.eks_oidc.arn + namespace_service_accounts = ["devlake:external-secret-devlake"] + } + } + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "external_secrets_role_attach_devlake" { + count = local.secret_count + + role = module.external_secrets_role_devlake[0].iam_role_name + policy_arn = aws_iam_policy.external-secrets-devlake[0].arn +} + +resource "kubernetes_manifest" "namespace_keycloak" { + count = local.secret_count + + manifest = { + "apiVersion" = "v1" + "kind" = "Namespace" + "metadata" = { + "name" = "keycloak" + } + } +} + +resource "kubernetes_manifest" "serviceaccount_external_secret_devlake" { + count = local.secret_count + depends_on = [ + kubernetes_manifest.namespace_devlake + ] + + manifest = { + "apiVersion" = "v1" + "kind" = "ServiceAccount" + "metadata" = { + "annotations" = { + "eks.amazonaws.com/role-arn" = tostring(module.external_secrets_role_devlake[0].iam_role_arn) + } + "name" = "external-secret-devlake" + "namespace" = "devlake" + } + } +} + +resource "kubectl_manifest" "keycloak_secret_store" { + depends_on = [ + kubernetes_manifest.serviceaccount_external_secret_devlake + ] + + yaml_body = templatefile("${path.module}/templates/manifests/devlake-secret-store.yaml", { + REGION = local.region + } + ) +} + + +resource "terraform_data" "devlake_encryption_secret" { + provisioner "local-exec" { + command = "openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1" + interpreter = ["/bin/bash", "-c"] + } +} + +resource "aws_secretsmanager_secret" "devlake_encryption_secret" { + count = local.secret_count + + description = "for use with modern engineering devlake installation" + name = "modern-engg/devlake/encryption" + recovery_window_in_days = 0 +} + +resource "aws_secretsmanager_secret_version" "devlake_encryption_secret" { + count = local.secret_count + + secret_id = aws_secretsmanager_secret.devlake_encryption_secret[0].id + secret_string = jsonencode({ + ENCRYPTION_SECRET = terraform_data.devlake_encryption_secret.result + }) +} + +resource "kubectl_manifest" "application_argocd_devlake" { + depends_on = [ + kubectl_manifest.devlake_secret_store + ] + + yaml_body = templatefile("${path.module}/templates/argocd-apps/devlake.yaml", { + GITHUB_URL = local.repo_url + } + ) + +} + + +resource "kubectl_manifest" "ingress_devlake" { + depends_on = [ + kubectl_manifest.application_argocd_devlake, + ] + + yaml_body = templatefile("${path.module}/templates/manifests/ingress-devlake.yaml", { + DEVLAKE_DOMAIN_NAME = local.domain_name + } + ) +} + diff --git a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml new file mode 100644 index 000000000..6911d9ddf --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml @@ -0,0 +1,29 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: crossplane + namespace: devlake + labels: + env: dev + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: modern-engg + sources: + - chart: devlake + repoURL: https://apache.github.io/incubator-devlake-helm-chart + targetRevision: 1.0.2-beta6 + helm: + releaseName: devlake + valueFiles: + - $values/packages/devlake/dev/values.yaml + - repoURL: ${GITHUB_URL} + targetRevision: HEAD + ref: values + destination: + server: "https://kubernetes.default.svc" + namespace: devlake + syncPolicy: + automated: {} + syncOptions: + - CreateNamespace=true diff --git a/platform/infra/terraform/mgmt/terraform/templates/manifests/devlake-secret-store.yaml b/platform/infra/terraform/mgmt/terraform/templates/manifests/devlake-secret-store.yaml new file mode 100644 index 000000000..a15d2abbf --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/templates/manifests/devlake-secret-store.yaml @@ -0,0 +1,14 @@ +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: devlake + namespace: devlake +spec: + provider: + aws: + service: SecretsManager + region: ${REGION} + auth: + jwt: + serviceAccountRef: + name: external-secret-devlake diff --git a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml new file mode 100644 index 000000000..e69de29bb From 25263e933cb98973802bed25d1fe091eed61e2d4 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Fri, 7 Mar 2025 13:34:32 -0800 Subject: [PATCH 04/63] devlake integ work --- packages/argo-rollouts/dev/cm-notifs.yaml | 18 + packages/argo-rollouts/dev/test-rollout.yaml | 8 + packages/devlake/base/values.yaml | 41 ++ packages/devlake/dev/kustomization.yaml | 5 +- .../dev/patches/deployment-devlake-ui.yaml | 11 - .../dev/patches/deployment-devlake.yaml | 11 - packages/devlake/dev/patches/devlake.yaml | 12 + packages/devlake/dev/values.yaml | 420 ------------------ packages/grafana/manifests/devlake.yaml | 11 + .../backstage/templates/catalog-info.yaml | 1 + .../template-devlake-onboarding.yaml | 0 .../manifests/events.yaml | 200 +++++++++ .../infra/terraform/mgmt/terraform/devlake.tf | 6 +- .../templates/argocd-apps/devlake.yaml | 3 +- .../templates/manifests/ingress-devlake.yaml | 23 + 15 files changed, 321 insertions(+), 449 deletions(-) create mode 100644 packages/argo-rollouts/dev/cm-notifs.yaml create mode 100644 packages/argo-rollouts/dev/test-rollout.yaml create mode 100644 packages/devlake/base/values.yaml delete mode 100644 packages/devlake/dev/patches/deployment-devlake-ui.yaml delete mode 100644 packages/devlake/dev/patches/deployment-devlake.yaml create mode 100644 packages/devlake/dev/patches/devlake.yaml delete mode 100644 packages/devlake/dev/values.yaml create mode 100644 packages/grafana/manifests/devlake.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml diff --git a/packages/argo-rollouts/dev/cm-notifs.yaml b/packages/argo-rollouts/dev/cm-notifs.yaml new file mode 100644 index 000000000..19e9cbec7 --- /dev/null +++ b/packages/argo-rollouts/dev/cm-notifs.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: argo-rollouts-notification-configmap + namespace: argo-rollouts +data: + trigger.on-rollout-aborted: | + - send: [modengg-rollout-aborted] + trigger.on-rollout-completed: | + - send: [modengg-rollout-completed] + service.webhook.deployments: | + url: https://${DNS_ENGINEERING}/argo-events/deployments + template.modengg-rollout-aborted: | + - stuff + template.modengg-rollout-completed: | + - stuff + + diff --git a/packages/argo-rollouts/dev/test-rollout.yaml b/packages/argo-rollouts/dev/test-rollout.yaml new file mode 100644 index 000000000..75f0b1973 --- /dev/null +++ b/packages/argo-rollouts/dev/test-rollout.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: test-rollout + annotations: + notifications.argoproj.io/subscribe.on-rollout-completed.deployments: "" + notifications.argoproj.io/subscribe.on-rollout-aborted.deployments: "" diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml new file mode 100644 index 000000000..27417d07a --- /dev/null +++ b/packages/devlake/base/values.yaml @@ -0,0 +1,41 @@ +# dependency chart values +grafana: + enabled: false + external: + url: "http://localhost:4000" # Set to AMG Endpoint in setup-environments + +mysql: + storage: + class: gp3 + +lake: + image: + repository: public.ecr.aws/o3w6s6n3/devlake/lake + pullPolicy: Always + tag: latest + # storage for config + encryptionSecret: + # The name of secret which contains keys named ENCRYPTION_SECRET + secretName: devlake-encryption-secret + + +ui: + image: + repository: public.ecr.aws/o3w6s6n3/devlake/ui + pullPolicy: Always + tag: latest + + # Look into adding an auth proxy like done: https://github.com/oauth2-proxy/oauth2-proxy/blob/master/contrib/local-environment/docker-compose-keycloak.yaml + basicAuth: + enabled: true + user: admin + password: devlake-admin + autoCreateSecret: true + secretName: "devlake-admin-secret" + +service: + # service type: NodePort/ClusterIP + type: ClusterIP + +ingress: + enabled: false diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 9feedb128..42ea812b8 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -1,3 +1,2 @@ -patchesStrategicMerge: - - patches/deployment-devlake-ui - - pathes/deployment-devlake +resources: + - external-secrets.yaml diff --git a/packages/devlake/dev/patches/deployment-devlake-ui.yaml b/packages/devlake/dev/patches/deployment-devlake-ui.yaml deleted file mode 100644 index 390e2188f..000000000 --- a/packages/devlake/dev/patches/deployment-devlake-ui.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: devlake-ui - namespace: devlake -spec: - template: - spec: - containers: - - name: devlake - image: ecr-one diff --git a/packages/devlake/dev/patches/deployment-devlake.yaml b/packages/devlake/dev/patches/deployment-devlake.yaml deleted file mode 100644 index e56dde2c3..000000000 --- a/packages/devlake/dev/patches/deployment-devlake.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: devlake-lake - namespace: devlake -spec: - template: - spec: - containers: - - name: devlake - image: ecr-one diff --git a/packages/devlake/dev/patches/devlake.yaml b/packages/devlake/dev/patches/devlake.yaml new file mode 100644 index 000000000..d87a8169f --- /dev/null +++ b/packages/devlake/dev/patches/devlake.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: devlake +spec: + template: + spec: + containers: + name: devlake + env: + - name: GRAFANA_ENDPOINT + value: ${AMG_ENDPOINT} diff --git a/packages/devlake/dev/values.yaml b/packages/devlake/dev/values.yaml deleted file mode 100644 index 0d4f62b7d..000000000 --- a/packages/devlake/dev/values.yaml +++ /dev/null @@ -1,420 +0,0 @@ -imageTag: v1.0.2-beta6 - -# image pull secrets -imagePullSecrets: [] - -#the common environments for all pods except grafana, grafana needs to be set in grafana section seperately -commonEnvs: - TZ: "UTC" - -mysql: - # if use external mysql server, please set true - # by default using false, chart will create a single mysql instance - useExternal: false - - # the external mysql server address - externalServer: 127.0.0.1 - - # external mysql port - externalPort: 3306 - - # the username for devlake database - username: merico - - # the password for devlake database - password: merico - - # the database for devlake - database: lake - - # root password for mysql, only used when use_external=false - rootPassword: admin - - # storage for mysql - storage: - # pvc or hostpath - type: pvc - # the storage class for pv, leave empty will using default - class: "" - size: 50Gi - hostPath: /devlake/mysql/data - - # image for mysql - image: - repository: mysql - tag: 8 - pullPolicy: IfNotPresent - - # init containers for mysql if have - initContainers: [] - - # resources config for mysql if have - resources: {} - - # nodeSelector config for mysql if have - nodeSelector: {} - - # tolerations config for mysql if have - tolerations: [] - - # affinity config for mysql if have - affinity: {} - - extraLabels: {} - - securityContext: {} - - containerSecurityContext: {} - - podAnnotations: {} - - service: - type: "ClusterIP" - nodePort: "" - loadBalancerIP: "" - extraLabels: {} - -# pgsql: -# # if use external pgsql server, please set true -# # by default using false, chart will create a single pgsql instance -# useExternal: false - -# # the external pgsql server address -# externalServer: 127.0.0.1 - -# # external pgsql port -# externalPort: 5432 -# # the username for devlake database -# username: merico - -# # the password for devlake database -# password: merico - -# # the database for devlake -# database: lake - -# # storage for pgsql -# storage: -# # the storage class for pv, leave empty will using default -# class: "" -# size: 5Gi - -# # image for pgsql -# image: -# repository: postgres -# tag: 14.5 -# pullPolicy: IfNotPresent - -# # resources config for pgsql if have -# resources: {} - -# # nodeSelector config for pgsql if have -# nodeSelector: {} - -# # tolerations config for pgsql if have -# tolerations: [] - -# # affinity config for pgsql if have -# affinity: {} - -# extraLabels: {} - -# securityContext: {} - -# containerSecurityContext: {} - -# annotations: {} - -# dependency chart values -grafana: - enabled: true - #if grafana enabled is false, then external url should be provided - external: - url: "" - image: - repository: devlake.docker.scarf.sh/apache/devlake-dashboard - tag: v1.0.2-beta6 - deploymentStrategy: - type: Recreate - adminPassword: "" - grafana.ini: - server: - serve_from_subpath: "true" - root_url: "%(protocol)s://%(domain)s/grafana" - #the secret name should be as same as .Values.option.connectionSecretName - envFromSecrets: - - name: "devlake-mysql-auth" - #keep grafana timezone same as other pods, which is set by .Values.commonEnvs.TZ - env: - TZ: "UTC" - persistence: - enabled: true - size: 4Gi - ingressServiceName: "" - ingressServicePort: "" - -lake: - image: - repository: devlake.docker.scarf.sh/apache/devlake - pullPolicy: Always - # defaults to imageTag; if set, lake.image.tag will override imageTag - # tag: - # storage for config - port: 8080 - envs: - API_TIMEOUT: "120s" - API_RETRY: "3" - API_REQUESTS_PER_HOUR: "10000" - PIPELINE_MAX_PARALLEL: "1" - IN_SECURE_SKIP_VERIFY: "false" - LOGGING_DIR: "/app/logs" - # debug, info, warn, error - LOGGING_LEVEL: "info" - JIRA_JQL_AUTO_FULL_REFRESH: "true" - ########################## - # ENABLE_SUBTASKS_BY_DEFAULT: This environment variable is used to enable or disable the execution of subtasks. - # The format is as follows: plugin_name1:subtask_name1:enabled_value,plugin_name2:subtask_name2:enabled_value,plugin_name3:subtask_name3:enabled_value - ########################## - # ENABLE_SUBTASKS_BY_DEFAULT default value as below: - # ENABLE_SUBTASKS_BY_DEFAULT: "jira:collectIssueChangelogs:true,jira:extractIssueChangelogs:true,jira:convertIssueChangelogs:true,tapd:collectBugChangelogs:true,tapd:extractBugChangelogs:true,tapd:convertBugChangelogs:true,zentao:collectBugRepoCommits:true,zentao:extractBugRepoCommits:true,zentao:convertBugRepoCommits:true,zentao:collectStoryRepoCommits:true,zentao:extractStoryRepoCommits:true,zentao:convertStoryRepoCommits:true,zentao:collectTaskRepoCommits:true,zentao:extractTaskRepoCommits:true,zentao:convertTaskRepoCommits:true" - - #extra envs from an existing secret - extraEnvsFromSecret: "" - encryptionSecret: - # The name of secret which contains keys named ENCRYPTION_SECRET - secretName: "" - # if secretName is empty, secret should be set - # you can generate the encryption secret via cmd `openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1` - secret: "" - autoCreateSecret: true - - # If hostNetwork is true, then dnsPolicy is set to ClusterFirstWithHostNet - hostNetwork: false - - resources: {} - - strategy: - type: Recreate - - nodeSelector: {} - - tolerations: [] - - affinity: {} - - extraLabels: {} - - securityContext: {} - - containerSecurityContext: {} - - podAnnotations: {} - - livenessProbe: - httpGet: - path: /ping - port: 8080 - scheme: HTTP - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - - readinessProbe: - httpGet: - path: /ping - port: 8080 - scheme: HTTP - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - - deployment: - extraLabels: {} - - service: - extraLabels: {} - - # Additional volumes to include in the Pod. Example: - # - # volumes: - # - name: my-volume - # configMap: my-config-map - volumes: [] - - # Additional volume mounts to include in the Container. Example: - # - # volumeMounts: - # - name: test-volume - # mountPath: /opt/test_folder - # subPath: test_file.yaml - volumeMounts: [] - -ui: - image: - repository: devlake.docker.scarf.sh/apache/devlake-config-ui - pullPolicy: Always - # defaults to imageTag; if set, lake.image.tag will override imageTag - # tag: - resources: {} - - strategy: {} - - nodeSelector: {} - - tolerations: [] - - affinity: {} - - livenessProbe: - httpGet: - path: /health/ - port: 4000 - scheme: HTTP - failureThreshold: 5 - initialDelaySeconds: 15 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - - readinessProbe: - httpGet: - path: /health/ - port: 4000 - scheme: HTTP - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - - basicAuth: - enabled: false - user: admin - password: admin - autoCreateSecret: true - secretName: "" - - extraLabels: {} - - podAnnotations: {} - - ## SecurityContext holds pod-level security attributes and common container settings. - ## This defaults to non root user with uid 101 and gid 1000. *v1.PodSecurityContext false - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: - {} - # fsGroup: 101 - # runAsGroup: 1000 - # runAsNonRoot: true - # runAsUser: 101 - - ## K8s containers' Security Context - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container - containerSecurityContext: - {} - # allowPrivilegeEscalation: false - # capabilities: - # drop: - # - all - - deployment: - extraLabels: {} - - service: - extraLabels: {} - - ## Side Contaainer Configuration - extraContainers: [] -# - name: vault-agent -# image: vault:1.6.2 -# args: -# - agent -# - -config -# - /vault/config/config.hcl -# env: -# - name: VAULT_ADDR -# value: https://vault:8200 -# - name: VAULT_SKIP_VERIFY -# value: "false" -# - name: VAULT_CACERT -# value: /vault/tls/ca.crt - -# alpine image for some init containers -alpine: - image: - repository: alpine - tag: 3.16 - pullPolicy: IfNotPresent - -service: - # service type: NodePort/ClusterIP - type: NodePort - # node port for devlake-ui if NodePort is enabled - uiPort: 32001 - -ingress: - enabled: false - enableHttps: false - # Set to false if you want to use a different ingress controller - useDefaultNginx: false - # ingress class name, example: alb for AWS load balancer controller - className: - # domain name for hosting devlake, must be set if ingress is enabled - hostname: localhost - extraLabels: {} - # annotations required for your ingress controller; see the examples below - # for nginx, use the first two lines of annotations - # for alb (w/ external-dns), use the last 5 (6) lines of annotations - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - # - # alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - # alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:xxx:certificate/xxx-xxx-xxx - # alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]' - # alb.ingress.kubernetes.io/scheme: internet-facing - # alb.ingress.kubernetes.io/target-type: ip - # external-dns.alpha.kubernetes.io/hostname: www.example.com - - # url prefix, not works right now, keep "/" - prefix: / - # if using https provides the certificates secret name - tlsSecretName: "" - # ingress http port - httpPort: 80 - # ingress https port - httpsPort: 443 - - extraPaths: [] -# extraPaths: -# - path: /* -# pathType: ImplementationSpecific -# backend: -# service: -# name: ssl-redirect -# port: -# name: use-annotation - -option: - # database type, supported: [mysql] - database: mysql - # the existing k8s secret name of db connection auth. The secret name should be as same as .Values.grafana.envFromSecret - connectionSecretName: "devlake-mysql-auth" - autoCreateSecret: true - -# Define some extra resources to be created -# This section is useful when you need ExternalResource or Secrets, etc. -extraResources: [] - # - apiVersion: v1 - # kind: Secret - # metadata: - # name: example-secret - # labels: {} - # type: Opaque - # stringData: - # username: admin - # password: mypassword diff --git a/packages/grafana/manifests/devlake.yaml b/packages/grafana/manifests/devlake.yaml new file mode 100644 index 000000000..f1ead991d --- /dev/null +++ b/packages/grafana/manifests/devlake.yaml @@ -0,0 +1,11 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: devlake-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards" diff --git a/platform/backstage/templates/catalog-info.yaml b/platform/backstage/templates/catalog-info.yaml index 43a0221f1..97f04de14 100644 --- a/platform/backstage/templates/catalog-info.yaml +++ b/platform/backstage/templates/catalog-info.yaml @@ -22,3 +22,4 @@ spec: - ./create-dev-and-prod-env/template-create-dev-and-prod-env.yaml - ./cicd-pipeline/template-cicd-pipeline.yaml - ./create-env-aurora-postgres/template-env-aurora-postgres.yaml + - ./devlake-onboarding/template-devlake-onboarding.yaml diff --git a/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml b/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml new file mode 100644 index 000000000..b9be5b4ed --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml @@ -0,0 +1,200 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-eventsource-svc +spec: + type: NodePort # Or ClusterIP if using Ingress + ports: + - port: 12000 + targetPort: 12000 + protocol: TCP + selector: + eventsource-name: gitea-webhook + +--- +apiVersion: argoproj.io/v1alpha1 +kind: EventBus +metadata: + name: default +spec: + nats: + native: + replicas: 3 + auth: token +--- +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: gitea-webhook + labels: + eventsource-name: gitea-webhook +spec: + service: + ports: + - port: 12000 + targetPort: 12000 + webhook: + incidents-webhook: + port: "12000" + endpoint: /webhook/incidents + method: POST + deployments-webhook: + port: "12000" + endpoint: /webhook/deployments + method: POST + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: gitea-webhook-sensor +spec: + template: + serviceAccountName: wf-sa + dependencies: + - name: incidents-webhook + eventSourceName: gitea-webhook + eventName: incidents-webhook + - name: deployments-webhook + eventSourceName: gitea-webhook + eventName: deployments-webhook + triggers: + - template: + name: incidents-workflow-trigger + conditions: "incidents-webhook" + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: incidents-workflow- + spec: + entrypoint: process-incident + serviceAccountName: wf-sa + arguments: + parameters: + - name: body + value: '{{.Input.body}}' + - name: headers + value: '{{.Input.headers}}' + - name: url + value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/issues + - name: token + value: ubr09ow9z27sfRldMCX3c7o1hxp3B4VvMUok2BJWV2rHhhtsXIfCQwhTaaUjbzlLEU8arHVMzMQ8bVjcnHWgQZ7Z2DEXHkUqfNjO0pxX6759rR26ee1ucflFEn0SshS4 + templates: + - name: process-incident + inputs: + parameters: + - name: body + - name: headers + - name: url + - name: token + script: + image: alpine + command: [sh] + source: | + apk add --no-cache jq curl + echo '{{inputs.parameters.body}}' + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + { + issueKey: (.repository.full_name + "-" + (.issue.number | tostring)), + title: .issue.title, + description: .issue.body, + url: .issue.html_url, + type: "INCIDENT", + status: (if .issue.state == "open" then "TODO" else "DONE" end), + originalStatus: "TODO", + createdDate: .issue.created_at, + updatedDate: .issue.updated_at, + resolutionDate: (if .issue.state == "closed" then .issue.closed_at else null end), + priority: "", + severity: "", + creatorId: .issue.user.login, + creatorName: (.issue.user.full_name // .issue.user.login), + assigneeId: (.issue.assignee.login // ""), + assigneeName: (if .issue.assignee then (.issue.assignee.full_name // .issue.assignee.login) else "" end) + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + TOKEN='{{inputs.parameters.token}}' + + curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + parameters: + - src: + dependencyName: incidents-webhook + dataKey: body + dest: spec.arguments.parameters.0.value + + - template: + name: deployments-workflow-trigger + conditions: "deployments-webhook" + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: deployments-workflow- + spec: + entrypoint: process-deployment + serviceAccountName: wf-sa + arguments: + parameters: + - name: body + value: '{{.Input.body}}' + - name: headers + value: '{{.Input.headers}}' + - name: url + value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/deployments + - name: token + value: ubr09ow9z27sfRldMCX3c7o1hxp3B4VvMUok2BJWV2rHhhtsXIfCQwhTaaUjbzlLEU8arHVMzMQ8bVjcnHWgQZ7Z2DEXHkUqfNjO0pxX6759rR26ee1ucflFEn0SshS4 + templates: + - name: process-deployment + inputs: + parameters: + - name: body + - name: headers + - name: url + - name: token + script: + image: alpine + command: [sh] + source: | + apk add --no-cache jq curl + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + . as $root | + { + id: .after, + createdDate: (.commits | first | .timestamp), + startedDate: (.commits | first | .timestamp), + finishedDate: (.commits | last | .timestamp), + environment: "PRODUCTION", + result: "SUCCESS", + name: ("deployment-" + (.after | .[0:7])), + deploymentCommits: [ + .commits[] | { + repoUrl: $root.repository.clone_url, + refName: $root.ref, + startedDate: .timestamp, + finishedDate: .timestamp, + commitSha: .id, + commitMsg: .message, + result: "SUCCESS", + name: ("deployment-" + (.id | .[0:7])) + } + ] + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + TOKEN='{{inputs.parameters.token}}' + + curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + parameters: + - src: + dependencyName: deployments-webhook + dataKey: body + dest: spec.arguments.parameters.0.value + diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index fcba4a4e9..3b8c76bf9 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -47,14 +47,14 @@ resource "aws_iam_role_policy_attachment" "external_secrets_role_attach_devlake" policy_arn = aws_iam_policy.external-secrets-devlake[0].arn } -resource "kubernetes_manifest" "namespace_keycloak" { +resource "kubernetes_manifest" "namespace_devlake" { count = local.secret_count manifest = { "apiVersion" = "v1" "kind" = "Namespace" "metadata" = { - "name" = "keycloak" + "name" = "devlake" } } } @@ -78,7 +78,7 @@ resource "kubernetes_manifest" "serviceaccount_external_secret_devlake" { } } -resource "kubectl_manifest" "keycloak_secret_store" { +resource "kubectl_manifest" "devlake_secret_store" { depends_on = [ kubernetes_manifest.serviceaccount_external_secret_devlake ] diff --git a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml index 6911d9ddf..70ba2704f 100644 --- a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml +++ b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml @@ -16,10 +16,11 @@ spec: helm: releaseName: devlake valueFiles: - - $values/packages/devlake/dev/values.yaml + - $values/packages/devlake/base/values.yaml - repoURL: ${GITHUB_URL} targetRevision: HEAD ref: values + path: packages/devlake/dev destination: server: "https://kubernetes.default.svc" namespace: devlake diff --git a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml index e69de29bb..6a425ccd1 100644 --- a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml +++ b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: devlake-ingress + namespace: devlake + annotations: + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/use-regex: 'true' +spec: + ingressClassName: nginx + rules: + - host: ${DEVLAKE_DOMAIN_NAME} + http: + paths: + - path: /devlake(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: devlake-ui + port: + number: 4000 From 9d6c457d25bcb52713360246c1cf050a879c2a44 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 17 Mar 2025 13:05:04 -0700 Subject: [PATCH 05/63] devlake fixes --- packages/devlake/base/values.yaml | 1 + platform/infra/terraform/mgmt/terraform/devlake.tf | 13 +++++++------ .../terraform/templates/argocd-apps/devlake.yaml | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 27417d07a..68e0468aa 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -17,6 +17,7 @@ lake: encryptionSecret: # The name of secret which contains keys named ENCRYPTION_SECRET secretName: devlake-encryption-secret + autoCreateSecret: false ui: diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 3b8c76bf9..8e9a8e1c5 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -90,11 +90,12 @@ resource "kubectl_manifest" "devlake_secret_store" { } -resource "terraform_data" "devlake_encryption_secret" { - provisioner "local-exec" { - command = "openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1" - interpreter = ["/bin/bash", "-c"] - } +resource "random_password" "devlake_encryption_secret" { + length = 128 + special = false + lower = false + upper = true + numeric = false } resource "aws_secretsmanager_secret" "devlake_encryption_secret" { @@ -110,7 +111,7 @@ resource "aws_secretsmanager_secret_version" "devlake_encryption_secret" { secret_id = aws_secretsmanager_secret.devlake_encryption_secret[0].id secret_string = jsonencode({ - ENCRYPTION_SECRET = terraform_data.devlake_encryption_secret.result + ENCRYPTION_SECRET = random_password.devlake_encryption_secret.result }) } diff --git a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml index 70ba2704f..d5b3410c7 100644 --- a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml +++ b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml @@ -1,8 +1,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: crossplane - namespace: devlake + name: devlake + namespace: argocd labels: env: dev finalizers: @@ -18,7 +18,7 @@ spec: valueFiles: - $values/packages/devlake/base/values.yaml - repoURL: ${GITHUB_URL} - targetRevision: HEAD + targetRevision: feat/dora-metrics ref: values path: packages/devlake/dev destination: From c6499c06660b12d54ea51a5dfee5507c2ae79c20 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 17 Mar 2025 14:17:03 -0700 Subject: [PATCH 06/63] add env to ui fix value --- packages/devlake/base/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 68e0468aa..1f2fff0aa 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -3,6 +3,9 @@ grafana: enabled: false external: url: "http://localhost:4000" # Set to AMG Endpoint in setup-environments +commonEnvs: + DEVLAKE_PATH_PREFIX: "/devlake" + mysql: storage: From 206e1a5ae150357dc160a5c75a06243f6a52314b Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 18 Mar 2025 12:36:09 -0700 Subject: [PATCH 07/63] get rid of ingress --- packages/devlake/base/values.yaml | 3 --- .../infra/terraform/mgmt/terraform/devlake.tf | 12 ---------- .../templates/manifests/ingress-devlake.yaml | 23 ------------------- 3 files changed, 38 deletions(-) delete mode 100644 platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 1f2fff0aa..68e0468aa 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -3,9 +3,6 @@ grafana: enabled: false external: url: "http://localhost:4000" # Set to AMG Endpoint in setup-environments -commonEnvs: - DEVLAKE_PATH_PREFIX: "/devlake" - mysql: storage: diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 8e9a8e1c5..21e63773d 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -127,15 +127,3 @@ resource "kubectl_manifest" "application_argocd_devlake" { } - -resource "kubectl_manifest" "ingress_devlake" { - depends_on = [ - kubectl_manifest.application_argocd_devlake, - ] - - yaml_body = templatefile("${path.module}/templates/manifests/ingress-devlake.yaml", { - DEVLAKE_DOMAIN_NAME = local.domain_name - } - ) -} - diff --git a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml deleted file mode 100644 index 6a425ccd1..000000000 --- a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: devlake-ingress - namespace: devlake - annotations: - nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/rewrite-target: /$2 - nginx.ingress.kubernetes.io/use-regex: 'true' -spec: - ingressClassName: nginx - rules: - - host: ${DEVLAKE_DOMAIN_NAME} - http: - paths: - - path: /devlake(/|$)(.*) - pathType: ImplementationSpecific - backend: - service: - name: devlake-ui - port: - number: 4000 From b94850e80ffde1b902254de87c191d90ed036983 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 18 Mar 2025 12:52:46 -0700 Subject: [PATCH 08/63] fix kyverno fix mysql more security fixes fix devlake user fix drop all --- packages/devlake/base/values.yaml | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 68e0468aa..b3c9d9b5b 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -7,6 +7,18 @@ grafana: mysql: storage: class: gp3 + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault lake: image: @@ -18,13 +30,48 @@ lake: # The name of secret which contains keys named ENCRYPTION_SECRET secretName: devlake-encryption-secret autoCreateSecret: false + securityContext: + runAsNonRoot: true + runAsUser: 1010 + runAsGroup: 1010 + fsGroup: 1010 + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault +alpine: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + ui: image: repository: public.ecr.aws/o3w6s6n3/devlake/ui pullPolicy: Always tag: latest + securityContext: + runAsNonRoot: true + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + # Look into adding an auth proxy like done: https://github.com/oauth2-proxy/oauth2-proxy/blob/master/contrib/local-environment/docker-compose-keycloak.yaml basicAuth: From 40cd482f5e68561448f6366c572640404f735d88 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 18 Mar 2025 15:00:22 -0700 Subject: [PATCH 09/63] export mysql for grafana --- packages/devlake/base/values.yaml | 31 +++++++++++++++++++ packages/devlake/dev/patches/devlake.yaml | 12 ------- .../infra/terraform/mgmt/terraform/devlake.tf | 15 ++++++++- .../templates/argocd-apps/devlake.yaml | 2 +- .../manifests/ingress-devlake-mysql.yaml | 24 ++++++++++++++ 5 files changed, 70 insertions(+), 14 deletions(-) delete mode 100644 packages/devlake/dev/patches/devlake.yaml create mode 100644 platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index b3c9d9b5b..d9bf09f98 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -7,11 +7,13 @@ grafana: mysql: storage: class: gp3 + securityContext: runAsNonRoot: true runAsUser: 999 runAsGroup: 999 fsGroup: 999 + containerSecurityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -20,6 +22,30 @@ mysql: seccompProfile: type: RuntimeDefault + initContainers: + - name: create-grafana-user + image: mysql:8 + command: + - /bin/bash + - -c + - | + until mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do + echo "Waiting for MySQL to be ready..." + sleep 2 + done + + echo "Creating Grafana user..." + mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " + CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password'; + GRANT SELECT ON lake.* TO 'grafanaReader'; + FLUSH PRIVILEGES; + " + + echo "User creation completed." + envFrom: + - secretRef: + name: devlake-mysql-auth + lake: image: repository: public.ecr.aws/o3w6s6n3/devlake/lake @@ -30,11 +56,13 @@ lake: # The name of secret which contains keys named ENCRYPTION_SECRET secretName: devlake-encryption-secret autoCreateSecret: false + securityContext: runAsNonRoot: true runAsUser: 1010 runAsGroup: 1010 fsGroup: 1010 + containerSecurityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -49,6 +77,7 @@ alpine: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + containerSecurityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -62,8 +91,10 @@ ui: repository: public.ecr.aws/o3w6s6n3/devlake/ui pullPolicy: Always tag: latest + securityContext: runAsNonRoot: true + containerSecurityContext: allowPrivilegeEscalation: false runAsNonRoot: true diff --git a/packages/devlake/dev/patches/devlake.yaml b/packages/devlake/dev/patches/devlake.yaml deleted file mode 100644 index d87a8169f..000000000 --- a/packages/devlake/dev/patches/devlake.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: devlake -spec: - template: - spec: - containers: - name: devlake - env: - - name: GRAFANA_ENDPOINT - value: ${AMG_ENDPOINT} diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 21e63773d..77be66e82 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -121,9 +121,22 @@ resource "kubectl_manifest" "application_argocd_devlake" { ] yaml_body = templatefile("${path.module}/templates/argocd-apps/devlake.yaml", { - GITHUB_URL = local.repo_url + GITHUB_URL = local.repo_url + GITHUB_BRANCH = "feat/dora-metrics" #TODO:SWITCH BEFORE PR } ) } +resource "kubectl_manifest" "ingress_devlake_mysql" { + depends_on = [ + kubectl_manifest.application_argocd_devlake, + ] + + yaml_body = templatefile("${path.module}/templates/manifests/ingress-devlake-mysql.yaml", { + DEVLAKE_DOMAIN_NAME = local.domain_name + } + ) +} + + diff --git a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml index d5b3410c7..2df501d1e 100644 --- a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml +++ b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/devlake.yaml @@ -18,7 +18,7 @@ spec: valueFiles: - $values/packages/devlake/base/values.yaml - repoURL: ${GITHUB_URL} - targetRevision: feat/dora-metrics + targetRevision: ${GITHUB_BRANCH} ref: values path: packages/devlake/dev destination: diff --git a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml new file mode 100644 index 000000000..5be719aa6 --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: devlake-mysql-metrics + namespace: devlake + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "MYSQL" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "60" + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" + nginx.ingress.kubernetes.io/ssl-redirect: "false" +spec: + ingressClassName: nginx + rules: + - host: ${DEVLAKE_DOMAIN_NAME} + http: + paths: + - path: /devlake/metrics + pathType: Prefix + backend: + service: + name: devlake-mysql + port: + number: 3306 + From 1d68307e01254461595b421fd95e06ec6d8ba50f Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 18 Mar 2025 15:07:23 -0700 Subject: [PATCH 10/63] fixes for mysql --- packages/devlake/base/values.yaml | 23 ---------------- packages/devlake/dev/kustomization.yaml | 1 + packages/devlake/dev/mysql-init-job.yaml | 34 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 23 deletions(-) create mode 100644 packages/devlake/dev/mysql-init-job.yaml diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index d9bf09f98..927e5b438 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -22,29 +22,6 @@ mysql: seccompProfile: type: RuntimeDefault - initContainers: - - name: create-grafana-user - image: mysql:8 - command: - - /bin/bash - - -c - - | - until mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do - echo "Waiting for MySQL to be ready..." - sleep 2 - done - - echo "Creating Grafana user..." - mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " - CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password'; - GRANT SELECT ON lake.* TO 'grafanaReader'; - FLUSH PRIVILEGES; - " - - echo "User creation completed." - envFrom: - - secretRef: - name: devlake-mysql-auth lake: image: diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 42ea812b8..4e374eab6 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -1,2 +1,3 @@ resources: - external-secrets.yaml + - mysql-init-job.yaml diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml new file mode 100644 index 000000000..c509f3d21 --- /dev/null +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: mysql-user-setup +spec: + template: + metadata: + name: mysql-user-setup + spec: + restartPolicy: OnFailure + containers: + - name: create-grafana-user + image: mysql:8 + command: + - /bin/bash + - -c + - | + until mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do + echo "Waiting for MySQL to be ready..." + sleep 2 + done + + echo "Creating Grafana user..." + mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " + CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password'; + GRANT SELECT ON lake.* TO 'grafanaReader'; + FLUSH PRIVILEGES; + " + + echo "User creation completed." + envFrom: + - secretRef: + name: devlake-mysql-auth + From 6dddfd148590395411c9a66e8cf9f4453497dad5 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 19 Mar 2025 09:36:14 -0700 Subject: [PATCH 11/63] more changes for grafana and devlake switch mysql to nodeport --- packages/devlake/base/values.yaml | 4 ++ packages/grafana/manifests/devlake.yaml | 27 ++++++++++++- packages/grafana/manifests/mysql.yaml | 38 +++++++++++++++++++ .../infra/terraform/setup-environments.sh | 20 +++++----- 4 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 packages/grafana/manifests/mysql.yaml diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 927e5b438..5a03514f7 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -8,6 +8,10 @@ mysql: storage: class: gp3 + service: + type: "NodePort" + nodePort: "30306" + securityContext: runAsNonRoot: true runAsUser: 999 diff --git a/packages/grafana/manifests/devlake.yaml b/packages/grafana/manifests/devlake.yaml index f1ead991d..423c68824 100644 --- a/packages/grafana/manifests/devlake.yaml +++ b/packages/grafana/manifests/devlake.yaml @@ -1,3 +1,28 @@ +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: dora-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: devlake-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" +--- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDashboard metadata: @@ -8,4 +33,4 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards" + url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" diff --git a/packages/grafana/manifests/mysql.yaml b/packages/grafana/manifests/mysql.yaml new file mode 100644 index 000000000..ba0c5adaf --- /dev/null +++ b/packages/grafana/manifests/mysql.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-mysql-creds + namespace: grafana-operator +type: Opaque +stringData: + GRAFANA_MYSQL_PASSWORD: "grafana_password" +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: mysql-datasource + namespace: grafana-operator +spec: + valuesFrom: + - targetPath: "secureJsonData.password" + valueFrom: + secretKeyRef: + name: "grafana-mysql-creds" + key: "GRAFANA_MYSQL_PASSWORD" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + datasource: + name: mysql + type: mysql + url: ${DNS}:3306 + user: grafanaReader + secureJsonData: + password: ${GRAFANA_MYSQL_PASSWORD} + jsonData: + database: lake + maxOpenConns: 100 + maxIdleConns: 100 + connMaxLifetime: 14400 + timeInterval: "1m" + tlsSkipVerify: true diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 2da83da98..1e64254a9 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -293,15 +293,15 @@ terraform -chdir=post-deploy apply -var aws_region="${TF_VAR_aws_region}" \ # Setup Applications on Clusters using ArgoCD on the management cluster # Substitute the GITHUB_URL and GITHUB_BRANCH in the manifests -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/ack-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ack-dev.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/ack-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ack-prod.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-dev.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-prod.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-dev.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-prod.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-dev.yaml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-prod.yaml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/ack-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ack-dev.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/ack-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ack-prod.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-dev.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-prod.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-dev.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-prod.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-dev.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-prod.yml # Apply the manifests to the management cluster kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ @@ -309,6 +309,8 @@ kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ # Setup Gitea Repo ${REPO_ROOT}/platform/infra/terraform/giteaInit.sh +# Add AMG URL to Devlake +kubectl set env deployment devlake-ui GRAFANA_ENDPOINT="https://$WORKSPACE_ENDPOINT/" -n devlake # Sleeping for Crossplane to be ready in DEV and PROD Cluster and restarting backstage pod kubectl rollout restart deployment backstage -n backstage sleep 120 From b37ecccf8f298b5ca53c08f1f585d4b2af93285e Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 19 Mar 2025 15:05:47 -0700 Subject: [PATCH 12/63] amg tf changes --- packages/grafana/manifests/mysql.yaml | 20 ++++++++++++++++--- platform/infra/terraform/bootstrap/main.tf | 7 ++++++- .../infra/terraform/bootstrap/variables.tf | 10 ++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/grafana/manifests/mysql.yaml b/packages/grafana/manifests/mysql.yaml index ba0c5adaf..acaa47c87 100644 --- a/packages/grafana/manifests/mysql.yaml +++ b/packages/grafana/manifests/mysql.yaml @@ -1,7 +1,16 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-mysql-endpoint # How to do programatically + namespace: grafana-operator +data: + MYSQL_ENDPOINT: NODEPORT_ENDPOINT +--- apiVersion: v1 kind: Secret metadata: - name: grafana-mysql-creds + name: grafana-mysql-creds # USE EXTERNAL-SECRETS namespace: grafana-operator type: Opaque stringData: @@ -18,14 +27,19 @@ spec: valueFrom: secretKeyRef: name: "grafana-mysql-creds" - key: "GRAFANA_MYSQL_PASSWORD" + key: GRAFANA_MYSQL_PASSWORD + - targetPath: "url" + valueFrom: + configMapRef: + name: "grafana-mysql-endpoint" + key: MYSQL_ENDPOINT instanceSelector: matchLabels: dashboards: "external-grafana" datasource: name: mysql type: mysql - url: ${DNS}:3306 + url: ${MYSQL_ENDPOINT} user: grafanaReader secureJsonData: password: ${GRAFANA_MYSQL_PASSWORD} diff --git a/platform/infra/terraform/bootstrap/main.tf b/platform/infra/terraform/bootstrap/main.tf index ab9c91359..bb60bceeb 100644 --- a/platform/infra/terraform/bootstrap/main.tf +++ b/platform/infra/terraform/bootstrap/main.tf @@ -99,5 +99,10 @@ module "managed_grafana" { # Dummy values for SAML configuration to setup will be updated after keycloak integration saml_idp_metadata_url = var.grafana_keycloak_idp_url + vpc_configuration = { + subnet_ids = var.vpc_private_subnets + security_group_ids = var.vpc_security_groups + } + tags = local.tags -} \ No newline at end of file +} diff --git a/platform/infra/terraform/bootstrap/variables.tf b/platform/infra/terraform/bootstrap/variables.tf index 870192e54..d8674cf4d 100644 --- a/platform/infra/terraform/bootstrap/variables.tf +++ b/platform/infra/terraform/bootstrap/variables.tf @@ -16,3 +16,13 @@ variable "grafana_keycloak_idp_url" { default = "http://modern-engg-xxxxxx.elb.us-west-2.amazonaws.com/keycloak/realms/grafana/protocol/saml/descriptor" } +variable "vpc_private_subnets" { + description = "VPC Private subnets for AMG configuration" + type = list(string) +} + +variable "vpc_security_groups" { + description = "VPC security groups for AMG configuration" + type = list(string) + +} From 546f5f65af66943a0ba9eca24568db19f82458d1 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 19 Mar 2025 15:43:42 -0700 Subject: [PATCH 13/63] changes for grafana --- packages/grafana/manifests/mysql.yaml | 2 +- platform/infra/terraform/bootstrap/main.tf | 4 ++-- .../infra/terraform/bootstrap/variables.tf | 6 ++--- .../grafana-workload-dashboards.yaml | 9 +++++++ .../infra/terraform/mgmt/terraform/devlake.tf | 12 ---------- .../mgmt/terraform/mgmt-cluster/outputs.tf | 5 ++++ .../manifests/ingress-devlake-mysql.yaml | 24 ------------------- .../infra/terraform/setup-environments.sh | 7 +++++- 8 files changed, 26 insertions(+), 43 deletions(-) delete mode 100644 platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml diff --git a/packages/grafana/manifests/mysql.yaml b/packages/grafana/manifests/mysql.yaml index acaa47c87..428afc553 100644 --- a/packages/grafana/manifests/mysql.yaml +++ b/packages/grafana/manifests/mysql.yaml @@ -5,7 +5,7 @@ metadata: name: grafana-mysql-endpoint # How to do programatically namespace: grafana-operator data: - MYSQL_ENDPOINT: NODEPORT_ENDPOINT + MYSQL_ENDPOINT: ${NODEPORT_ENDPOINT} --- apiVersion: v1 kind: Secret diff --git a/platform/infra/terraform/bootstrap/main.tf b/platform/infra/terraform/bootstrap/main.tf index bb60bceeb..e0844c12b 100644 --- a/platform/infra/terraform/bootstrap/main.tf +++ b/platform/infra/terraform/bootstrap/main.tf @@ -100,8 +100,8 @@ module "managed_grafana" { saml_idp_metadata_url = var.grafana_keycloak_idp_url vpc_configuration = { - subnet_ids = var.vpc_private_subnets - security_group_ids = var.vpc_security_groups + subnet_ids = var.eks_cluster_private_subnets + security_group_ids = [var.eks_cluster_node_security_group_id] } tags = local.tags diff --git a/platform/infra/terraform/bootstrap/variables.tf b/platform/infra/terraform/bootstrap/variables.tf index d8674cf4d..385c83774 100644 --- a/platform/infra/terraform/bootstrap/variables.tf +++ b/platform/infra/terraform/bootstrap/variables.tf @@ -16,13 +16,13 @@ variable "grafana_keycloak_idp_url" { default = "http://modern-engg-xxxxxx.elb.us-west-2.amazonaws.com/keycloak/realms/grafana/protocol/saml/descriptor" } -variable "vpc_private_subnets" { +variable "eks_cluster_private_subnets" { description = "VPC Private subnets for AMG configuration" type = list(string) } -variable "vpc_security_groups" { +variable "eks_cluster_node_security_group_id" { description = "VPC security groups for AMG configuration" - type = list(string) + type = string } diff --git a/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml b/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml index 030b52e74..8946ae021 100644 --- a/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml +++ b/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml @@ -9,6 +9,15 @@ spec: repoURL: ${GITHUB_URL} targetRevision: ${GITHUB_BRANCH} path: "packages/grafana/manifests/" + kustomize: + patches: + - target: + kind: ConfigMap + name: grafana-mysql-endpoint + patch: |- + - op: replace + path: /data/MYSQL_ENDPOINT + value: ${DEVLAKE_MYSQL_NODEPORT} destination: name: 'dev-cluster' namespace: grafana-operator diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 77be66e82..565426073 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -128,15 +128,3 @@ resource "kubectl_manifest" "application_argocd_devlake" { } -resource "kubectl_manifest" "ingress_devlake_mysql" { - depends_on = [ - kubectl_manifest.application_argocd_devlake, - ] - - yaml_body = templatefile("${path.module}/templates/manifests/ingress-devlake-mysql.yaml", { - DEVLAKE_DOMAIN_NAME = local.domain_name - } - ) -} - - diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf index fac829ea4..7fd801dc8 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf @@ -8,6 +8,11 @@ output "eks_cluster_private_subnets" { value = module.vpc.private_subnets } +output "eks_cluster_node_security_group_id" { + description = "EKS Cluster Node Security Group Id" + value = module.eks.node_security_group_id +} + output "vpc_cidr" { description = "Default VPC CIDR of the VPC created by the Module" value = local.vpc_cidr diff --git a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml b/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml deleted file mode 100644 index 5be719aa6..000000000 --- a/platform/infra/terraform/mgmt/terraform/templates/manifests/ingress-devlake-mysql.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: devlake-mysql-metrics - namespace: devlake - annotations: - nginx.ingress.kubernetes.io/backend-protocol: "MYSQL" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "60" - nginx.ingress.kubernetes.io/proxy-read-timeout: "60" - nginx.ingress.kubernetes.io/ssl-redirect: "false" -spec: - ingressClassName: nginx - rules: - - host: ${DEVLAKE_DOMAIN_NAME} - http: - paths: - - path: /devlake/metrics - pathType: Prefix - backend: - service: - name: devlake-mysql - port: - number: 3306 - diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 1e64254a9..0f8ade8cd 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -62,6 +62,7 @@ ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) +export TF_eks_cluster_node_security_group_id=$(terrafrom output -raw eks_cluster_node_security_group_id) echo "private subnets are : " $TF_eks_cluster_private_subnets # For Database and EC2 database export TF_eks_cluster_vpc_cidr=$(terraform output -raw vpc_cidr) @@ -290,6 +291,10 @@ terraform -chdir=post-deploy apply -var aws_region="${TF_VAR_aws_region}" \ -var dev_cluster_name="${TF_VAR_dev_cluster_name}" \ -var prod_cluster_name="${TF_VAR_prod_cluster_name}" -auto-approve +NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') +NODE_PORT=$(kubectl get service devlake-mysql -n devlake -o jsonpath='{.spec.ports[0].nodePort}') +export DEVLAKE_MYSQL_NODEPORT="${NODE_IP}:${NODE_PORT}" + # Setup Applications on Clusters using ArgoCD on the management cluster # Substitute the GITHUB_URL and GITHUB_BRANCH in the manifests @@ -299,7 +304,7 @@ sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANC sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-prod.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-dev.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-prod.yml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" -e "s#\${NODEPORT_ENDPOINT}#${DEVLAKE_MYSQL_NODEPORT}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-dev.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-prod.yml From 0f30b359384204428dd55b7d482ff5cb96fdb86f Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 19 Mar 2025 16:19:26 -0700 Subject: [PATCH 14/63] fix security group fix dashboards fix fix fix fix sed --- packages/grafana/manifests/devlake.yaml | 46 +++++++++---------- packages/grafana/manifests/kustomization.yaml | 5 ++ packages/grafana/manifests/mysql.yaml | 3 +- platform/infra/terraform/bootstrap/main.tf | 1 + .../mgmt/terraform/mgmt-cluster/outputs.tf | 2 +- .../infra/terraform/setup-environments.sh | 9 ++-- 6 files changed, 37 insertions(+), 29 deletions(-) create mode 100644 packages/grafana/manifests/kustomization.yaml diff --git a/packages/grafana/manifests/devlake.yaml b/packages/grafana/manifests/devlake.yaml index 423c68824..de8695028 100644 --- a/packages/grafana/manifests/devlake.yaml +++ b/packages/grafana/manifests/devlake.yaml @@ -11,26 +11,26 @@ spec: dashboards: "external-grafana" url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" --- -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: devlake-grafanadashboard - namespace: grafana-operator -spec: - folder: "Devlake Dashboards" - instanceSelector: - matchLabels: - dashboards: "external-grafana" - url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" ---- -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: devlake-grafanadashboard - namespace: grafana-operator -spec: - folder: "Devlake Dashboards" - instanceSelector: - matchLabels: - dashboards: "external-grafana" - url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" +# apiVersion: grafana.integreatly.org/v1beta1 +# kind: GrafanaDashboard +# metadata: +# name: devlake-grafanadashboard +# namespace: grafana-operator +# spec: +# folder: "Devlake Dashboards" +# instanceSelector: +# matchLabels: +# dashboards: "external-grafana" +# url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" +# --- +# apiVersion: grafana.integreatly.org/v1beta1 +# kind: GrafanaDashboard +# metadata: +# name: devlake-grafanadashboard +# namespace: grafana-operator +# spec: +# folder: "Devlake Dashboards" +# instanceSelector: +# matchLabels: +# dashboards: "external-grafana" +# url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" diff --git a/packages/grafana/manifests/kustomization.yaml b/packages/grafana/manifests/kustomization.yaml new file mode 100644 index 000000000..d42430162 --- /dev/null +++ b/packages/grafana/manifests/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - devlake.yaml + - mysql.yaml + - rust.yaml + diff --git a/packages/grafana/manifests/mysql.yaml b/packages/grafana/manifests/mysql.yaml index 428afc553..dff886ec3 100644 --- a/packages/grafana/manifests/mysql.yaml +++ b/packages/grafana/manifests/mysql.yaml @@ -30,7 +30,7 @@ spec: key: GRAFANA_MYSQL_PASSWORD - targetPath: "url" valueFrom: - configMapRef: + configMapKeyRef: name: "grafana-mysql-endpoint" key: MYSQL_ENDPOINT instanceSelector: @@ -40,6 +40,7 @@ spec: name: mysql type: mysql url: ${MYSQL_ENDPOINT} + access: proxy user: grafanaReader secureJsonData: password: ${GRAFANA_MYSQL_PASSWORD} diff --git a/platform/infra/terraform/bootstrap/main.tf b/platform/infra/terraform/bootstrap/main.tf index e0844c12b..639f7fd04 100644 --- a/platform/infra/terraform/bootstrap/main.tf +++ b/platform/infra/terraform/bootstrap/main.tf @@ -99,6 +99,7 @@ module "managed_grafana" { # Dummy values for SAML configuration to setup will be updated after keycloak integration saml_idp_metadata_url = var.grafana_keycloak_idp_url + create_security_group = false vpc_configuration = { subnet_ids = var.eks_cluster_private_subnets security_group_ids = [var.eks_cluster_node_security_group_id] diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf index 7fd801dc8..0a6641c9e 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf @@ -10,7 +10,7 @@ output "eks_cluster_private_subnets" { output "eks_cluster_node_security_group_id" { description = "EKS Cluster Node Security Group Id" - value = module.eks.node_security_group_id + value = module.eks.cluster_primary_security_group_id } output "vpc_cidr" { diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 0f8ade8cd..ace205c59 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -62,7 +62,7 @@ ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) -export TF_eks_cluster_node_security_group_id=$(terrafrom output -raw eks_cluster_node_security_group_id) +export TF_eks_cluster_node_security_group_id=$(terraform output -raw eks_cluster_node_security_group_id) echo "private subnets are : " $TF_eks_cluster_private_subnets # For Database and EC2 database export TF_eks_cluster_vpc_cidr=$(terraform output -raw vpc_cidr) @@ -122,8 +122,9 @@ git clone https://github.com/aws-observability/terraform-aws-observability-accel echo "bootstrapping Terraform" terraform -chdir=bootstrap init -reconfigure -terraform -chdir=bootstrap plan -terraform -chdir=bootstrap apply -auto-approve +terraform -chdir=bootstrap plan -var eks_cluster_private_subnets="$TF_eks_cluster_private_subnets" -var eks_cluster_node_security_group_id="$TF_eks_cluster_node_security_group_id" +terraform -chdir=bootstrap apply -var eks_cluster_private_subnets="$TF_eks_cluster_private_subnets" -var eks_cluster_node_security_group_id="$TF_eks_cluster_node_security_group_id" \ + -auto-approve export TF_VAR_state_s3_bucket=$(terraform -chdir=bootstrap output -raw eks-accelerator-bootstrap-state-bucket) export TF_VAR_state_ddb_lock_table=$(terraform -chdir=bootstrap output -raw eks-accelerator-bootstrap-ddb-lock-table) @@ -304,7 +305,7 @@ sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANC sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-prod.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-dev.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-prod.yml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" -e "s#\${NODEPORT_ENDPOINT}#${DEVLAKE_MYSQL_NODEPORT}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" -e "s#\${DEVLAKE_MYSQL_NODEPORT}#${DEVLAKE_MYSQL_NODEPORT}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-dev.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-prod.yml From d501b6def5f632063f5559c717b6b93c75861d62 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Fri, 21 Mar 2025 15:30:23 -0700 Subject: [PATCH 15/63] fix grafana security group and dora dashboards --- packages/grafana/manifests/devlake.yaml | 72 ++++++++++++------- .../mgmt/terraform/mgmt-cluster/outputs.tf | 2 +- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/packages/grafana/manifests/devlake.yaml b/packages/grafana/manifests/devlake.yaml index e62364152..08fbea6b0 100644 --- a/packages/grafana/manifests/devlake.yaml +++ b/packages/grafana/manifests/devlake.yaml @@ -9,28 +9,52 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://raw.githubusercontent.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORA.json" --- -# apiVersion: grafana.integreatly.org/v1beta1 -# kind: GrafanaDashboard -# metadata: -# name: devlake-grafanadashboard -# namespace: grafana-operator -# spec: -# folder: "Devlake Dashboards" -# instanceSelector: -# matchLabels: -# dashboards: "external-grafana" -# url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" -# --- -# apiVersion: grafana.integreatly.org/v1beta1 -# kind: GrafanaDashboard -# metadata: -# name: devlake-grafanadashboard -# namespace: grafana-operator -# spec: -# folder: "Devlake Dashboards" -# instanceSelector: -# matchLabels: -# dashboards: "external-grafana" -# url: "https://github.com/apache/incubator-devlake/tree/main/grafana/dashboards/DORA.json" +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: dora-deploymentfreq-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-DeploymentFrequency.json" +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: dora-changefailure-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-ChangeFailureRate.json" +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: dora-recoverytime-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json" +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: dora-leadtime-grafanadashboard + namespace: grafana-operator +spec: + folder: "Devlake Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-LeadTimeForChanges.json" diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf index 0a6641c9e..7fd801dc8 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf @@ -10,7 +10,7 @@ output "eks_cluster_private_subnets" { output "eks_cluster_node_security_group_id" { description = "EKS Cluster Node Security Group Id" - value = module.eks.cluster_primary_security_group_id + value = module.eks.node_security_group_id } output "vpc_cidr" { From be7bb7c50590aa2d20813970dcc0a2f79dfe828b Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 24 Mar 2025 15:37:22 -0700 Subject: [PATCH 16/63] start devlake init --- .../mgmt/terraform/scripts/devlake/install.sh | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh new file mode 100644 index 000000000..12183be4f --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh @@ -0,0 +1,95 @@ +# do db migration, add some data to devlake +kubectl wait --for=jsonpath=.status.health.status=Healthy -n argocd application/devlake +kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=60s +kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=60s +echo "Performing DB Migration" + +ADMIN_PASSWORD=$(kubectl get secret -n keycloak keycloak-config -o go-template='{{index .data "KEYCLOAK_ADMIN_PASSWORD" | base64decode}}') +kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & +pid=$! +trap '{ + kill $pid +}' EXIT +echo "waiting for port forward to be ready" +while ! nc -vz localhost 9090 >/dev/null 2>&1; do + sleep 2 +done + +curl -X POST localhost:9090/proceed-db-migration + +projectName="modengg" + +projectCreateRequest=$( + cat < Date: Tue, 25 Mar 2025 13:20:26 -0700 Subject: [PATCH 17/63] rename --- .../mgmt/terraform/scripts/devlake/{install.sh => init.sh} | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename platform/infra/terraform/mgmt/terraform/scripts/devlake/{install.sh => init.sh} (95%) diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh similarity index 95% rename from platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh rename to platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh index 12183be4f..a1f5f6f0b 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/install.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh @@ -4,7 +4,6 @@ kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --tim kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=60s echo "Performing DB Migration" -ADMIN_PASSWORD=$(kubectl get secret -n keycloak keycloak-config -o go-template='{{index .data "KEYCLOAK_ADMIN_PASSWORD" | base64decode}}') kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & pid=$! trap '{ @@ -53,6 +52,11 @@ EOF ) webhookCreateResponse=$(curl -X POST localhost:9090/plugins/webhook/connections -d "$webhookCreateRequest") webhookID=$(echo $webhookCreateResponse | jq -r '.id') +webhookApiKey=$(echo $webhookCreateResponse | jq -r '.apiKey.apiKey') +postIssuesEndpoint=$(echo $webhookCreateResponse | jq -r '.postIssuesEndpoint') +postPullRequestsEndpoint=$(echo $webhookCreateResponse | jq -r '.postPullRequestsEndpoint') +postPipelineDeployTaskEndpoint=$(echo $webhookCreateResponse | jq -r 'postPipelineDeployTaskEndpoint') + # {"name":"modengg_gitea","id":1,"createdAt":"2025-03-21T23:47:42.486Z","updatedAt":"2025-03-21T23:47:42.486Z","postIssuesEndpoint":"/rest/plugins/webhook/connections/1/issues","closeIssuesEndpoint":"/rest/plugins/webhook/connections/1/issue/:issueKey/close","postPullRequestsEndpoint":"/rest/plugins/webhook/connections/1/pull_requests","postPipelineTaskEndpoint":"/rest/plugins/webhook/connections/1/cicd_tasks","postPipelineDeployTaskEndpoint":"/rest/plugins/webhook/connections/1/deployments","closePipelineEndpoint":"/rest/plugins/webhook/connections/1/cicd_pipeline/:pipelineName/finish","apiKey":{"id":1,"createdAt":"2025-03-21T23:47:42.490396162Z","updatedAt":"2025-03-21T23:47:42.490396162Z","creator":"admin","creatorEmail":"","updater":"admin","updaterEmail":"","name":"webhook-1","apiKey":"fPaKeWtWhTUdtlpneoibhrtvwiMbBiPL3022Yyda30WrqrrBTX882NQ1b5uhLK4kzwQCMi4Qxl65m2Tfvc9h12nwR1btEmB2wbkMARbg0FQNTxAwFudqN9u7QB1atuLT","expiredAt":null,"allowedPath":"/plugins/webhook/connections/1/.*","type":"plugin:webhook","extra":"connectionId:1"}} blueprintPatchRequest=$( From 026d41a78867cc92fb470a5aa80cbcd4566300ff Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 25 Mar 2025 17:36:58 -0700 Subject: [PATCH 18/63] script fixes --- .../mgmt/terraform/scripts/devlake/init.sh | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) mode change 100644 => 100755 platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh old mode 100644 new mode 100755 index a1f5f6f0b..962c29eb1 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh @@ -2,19 +2,20 @@ kubectl wait --for=jsonpath=.status.health.status=Healthy -n argocd application/devlake kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=60s kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=60s -echo "Performing DB Migration" kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & pid=$! trap '{ kill $pid }' EXIT + echo "waiting for port forward to be ready" while ! nc -vz localhost 9090 >/dev/null 2>&1; do sleep 2 done -curl -X POST localhost:9090/proceed-db-migration +echo "Performing DB Migration" +curl -X GET localhost:9090/proceed-db-migration projectName="modengg" @@ -38,24 +39,28 @@ projectCreateRequest=$( } EOF ) -projectCreateResponse=$(curl -X POST localhost:9090/projects -d "$projectCreateRequest") +echo $projectCreateRequest +echo "Creating Project: $projectName" +projectCreateResponse=$(curl -X POST -H "Content-Type: application/json" localhost:9090/projects -d "$projectCreateRequest") +echo $projectCreateResponse blueprintID=$(echo $projectCreateResponse | jq -r '.blueprint.id') blueprintName=$(echo $projectCreateResponse | jq -r '.blueprint.name') # {"name":"modengg","description":"","createdAt":"2025-03-21T23:46:01.144Z","updatedAt":"2025-03-21T23:46:01.144Z","_raw_data_params":"","_raw_data_table":"","_raw_data_id":0,"_raw_data_remark":"","metrics":[{"pluginName":"dora","pluginOption":{},"enable":true},{"pluginName":"issue_trace","pluginOption":{},"enable":true}],"blueprint":{"name":"test-Blueprint","projectName":"test","mode":"NORMAL","plan":null,"enable":true,"cronConfig":"0 0 * * 1","isManual":false,"beforePlan":null,"afterPlan":null,"labels":[],"connections":[],"skipOnFail":false,"timeAfter":"2024-09-21T00:00:00Z","skipCollectors":false,"fullSync":false,"id":1,"createdAt":"2025-03-21T23:46:01.148Z","updatedAt":"2025-03-21T23:46:01.148Z"}} +echo "Creating Webhook" webhookCreateRequest=$( cat < Date: Wed, 2 Apr 2025 12:38:16 -0700 Subject: [PATCH 19/63] start backstage template for onboarding --- packages/grafana/manifests/devlake.yaml | 10 +- .../template-devlake-onboarding.yaml | 102 ++++++ .../manifests/events.yaml | 200 ----------- .../provisioner/generate-data-linux.sh | 185 +++++++++++ .../provisioner/generate-data.sh | 184 ++++++++++ .../provisioner/init.sh | 83 +++++ .../provisioner/rbac.yaml | 19 ++ .../provisioner/wf-run.yaml | 57 ++++ .../provisioner/wf-templates.yaml | 314 ++++++++++++++++++ .../skeleton/catalog-info.yaml | 22 ++ .../skeleton/manifests/bus.yaml | 0 .../skeleton/manifests/events.yaml | 152 +++++++++ .../skeleton/manifests/exit-handler.yaml | 0 .../manifests/gitea-external-secret.yaml | 25 ++ .../skeleton/manifests/rollout-notifs.yaml | 18 + .../skeleton/manifests/wf-templates.yaml | 175 ++++++++++ .../infra/terraform/mgmt/terraform/devlake.tf | 2 +- 17 files changed, 1342 insertions(+), 206 deletions(-) delete mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data-linux.sh create mode 100755 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data.sh create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/init.sh create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/rbac.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/catalog-info.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/bus.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml diff --git a/packages/grafana/manifests/devlake.yaml b/packages/grafana/manifests/devlake.yaml index 08fbea6b0..d996e7506 100644 --- a/packages/grafana/manifests/devlake.yaml +++ b/packages/grafana/manifests/devlake.yaml @@ -5,7 +5,7 @@ metadata: name: dora-grafanadashboard namespace: grafana-operator spec: - folder: "Devlake Dashboards" + folder: "DORA Dashboards" instanceSelector: matchLabels: dashboards: "external-grafana" @@ -17,7 +17,7 @@ metadata: name: dora-deploymentfreq-grafanadashboard namespace: grafana-operator spec: - folder: "Devlake Dashboards" + folder: "DORA Dashboards" instanceSelector: matchLabels: dashboards: "external-grafana" @@ -29,7 +29,7 @@ metadata: name: dora-changefailure-grafanadashboard namespace: grafana-operator spec: - folder: "Devlake Dashboards" + folder: "DORA Dashboards" instanceSelector: matchLabels: dashboards: "external-grafana" @@ -41,7 +41,7 @@ metadata: name: dora-recoverytime-grafanadashboard namespace: grafana-operator spec: - folder: "Devlake Dashboards" + folder: "DORA Dashboards" instanceSelector: matchLabels: dashboards: "external-grafana" @@ -53,7 +53,7 @@ metadata: name: dora-leadtime-grafanadashboard namespace: grafana-operator spec: - folder: "Devlake Dashboards" + folder: "DORA Dashboards" instanceSelector: matchLabels: dashboards: "external-grafana" diff --git a/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml b/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml index e69de29bb..c84392274 100644 --- a/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml +++ b/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml @@ -0,0 +1,102 @@ +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + description: Onboard an Application Team to DevLake + name: devlake-onboarding + title: Onboard App to DevLake +spec: + owner: guest + type: service + parameters: + - properties: + appname: + title: name of app for onboarding + type: string + required: + - appname + steps: + - id: fetchSystem + name: Fetch System + action: catalog:fetch + input: + entityRef: system:default/system-info + - action: fetch:template + name: Fetch Base + id: fetch-base + input: + url: ./skeleton/ + values: + appname: ${{ parameters.appname }} + namespace: team-${{ parameters.appname }} + hostname: ${{ steps['fetchSystem'].output.entity.spec.hostname }} + + - id: publish + name: Publishing to a gitea git repository + action: publish:gitea + input: + description: Devlake onboarding template + # Hard coded value for this demo purposes only. + repoUrl: ${{ steps['fetchSystem'].output.entity.spec.hostname }}/gitea?repo=${{parameters.appname}}-devlake + defaultBranch: main + + - action: fetch:template + name: Fetch Provisioner + id: fetch-provisioner + input: + url: ./provisioner/ + values: + appname: ${{ parameters.appname }} + namespace: team-${{ parameters.appname }} + hostname: ${{ steps['fetchSystem'].output.entity.spec.hostname }} + + - action: cnoe:kubernetes:apply + name: Apply Provisioner rbac + id: apply-prov-rbac + input: + namespaced: true + manifestPath: rbac.yaml + clusterName: local + + - action: cnoe:kubernetes:apply + name: Apply Workflow Templates + id: apply-temp + input: + namespaced: true + manifestPath: wf-templates.yaml + clusterName: local + + - action: cnoe:kubernetes:apply + name: Run Workflow + id: run-wf + input: + namespaced: true + manifestPath: wf-run.yaml + clusterName: local + + - id: create-argocd-app + name: Create ArgoCD App + action: cnoe:create-argocd-app + input: + appName: ${{parameters.appname}} + appNamespace: team-${{ parameters.appname }} + argoInstance: in-cluster + projectName: default + # necessary until we generate our own cert + repoUrl: http://my-gitea-http.gitea.svc.cluster.local:3000/giteaAdmin/${{parameters.appname}}-devlake + path: "manifests" + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }} + catalogInfoPath: 'catalog-info.yaml' + output: + links: + - title: Open in catalog + icon: catalog + entityRef: ${{ steps['register'].output.entityRef }} + - title: Open in argo workflows + url: https://${{ steps['fetchSystem'].output.entity.spec.hostname }}/argo-workflows/workflows/team-${{ parameters.appname }} + + + diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml deleted file mode 100644 index b9be5b4ed..000000000 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/manifests/events.yaml +++ /dev/null @@ -1,200 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: webhook-eventsource-svc -spec: - type: NodePort # Or ClusterIP if using Ingress - ports: - - port: 12000 - targetPort: 12000 - protocol: TCP - selector: - eventsource-name: gitea-webhook - ---- -apiVersion: argoproj.io/v1alpha1 -kind: EventBus -metadata: - name: default -spec: - nats: - native: - replicas: 3 - auth: token ---- -apiVersion: argoproj.io/v1alpha1 -kind: EventSource -metadata: - name: gitea-webhook - labels: - eventsource-name: gitea-webhook -spec: - service: - ports: - - port: 12000 - targetPort: 12000 - webhook: - incidents-webhook: - port: "12000" - endpoint: /webhook/incidents - method: POST - deployments-webhook: - port: "12000" - endpoint: /webhook/deployments - method: POST - ---- -apiVersion: argoproj.io/v1alpha1 -kind: Sensor -metadata: - name: gitea-webhook-sensor -spec: - template: - serviceAccountName: wf-sa - dependencies: - - name: incidents-webhook - eventSourceName: gitea-webhook - eventName: incidents-webhook - - name: deployments-webhook - eventSourceName: gitea-webhook - eventName: deployments-webhook - triggers: - - template: - name: incidents-workflow-trigger - conditions: "incidents-webhook" - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: incidents-workflow- - spec: - entrypoint: process-incident - serviceAccountName: wf-sa - arguments: - parameters: - - name: body - value: '{{.Input.body}}' - - name: headers - value: '{{.Input.headers}}' - - name: url - value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/issues - - name: token - value: ubr09ow9z27sfRldMCX3c7o1hxp3B4VvMUok2BJWV2rHhhtsXIfCQwhTaaUjbzlLEU8arHVMzMQ8bVjcnHWgQZ7Z2DEXHkUqfNjO0pxX6759rR26ee1ucflFEn0SshS4 - templates: - - name: process-incident - inputs: - parameters: - - name: body - - name: headers - - name: url - - name: token - script: - image: alpine - command: [sh] - source: | - apk add --no-cache jq curl - echo '{{inputs.parameters.body}}' - PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' - { - issueKey: (.repository.full_name + "-" + (.issue.number | tostring)), - title: .issue.title, - description: .issue.body, - url: .issue.html_url, - type: "INCIDENT", - status: (if .issue.state == "open" then "TODO" else "DONE" end), - originalStatus: "TODO", - createdDate: .issue.created_at, - updatedDate: .issue.updated_at, - resolutionDate: (if .issue.state == "closed" then .issue.closed_at else null end), - priority: "", - severity: "", - creatorId: .issue.user.login, - creatorName: (.issue.user.full_name // .issue.user.login), - assigneeId: (.issue.assignee.login // ""), - assigneeName: (if .issue.assignee then (.issue.assignee.full_name // .issue.assignee.login) else "" end) - }') - echo "====Transformed Payload====" - echo "$PAYLOAD" | jq '.' - TOKEN='{{inputs.parameters.token}}' - - curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' - parameters: - - src: - dependencyName: incidents-webhook - dataKey: body - dest: spec.arguments.parameters.0.value - - - template: - name: deployments-workflow-trigger - conditions: "deployments-webhook" - argoWorkflow: - operation: submit - source: - resource: - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: deployments-workflow- - spec: - entrypoint: process-deployment - serviceAccountName: wf-sa - arguments: - parameters: - - name: body - value: '{{.Input.body}}' - - name: headers - value: '{{.Input.headers}}' - - name: url - value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/deployments - - name: token - value: ubr09ow9z27sfRldMCX3c7o1hxp3B4VvMUok2BJWV2rHhhtsXIfCQwhTaaUjbzlLEU8arHVMzMQ8bVjcnHWgQZ7Z2DEXHkUqfNjO0pxX6759rR26ee1ucflFEn0SshS4 - templates: - - name: process-deployment - inputs: - parameters: - - name: body - - name: headers - - name: url - - name: token - script: - image: alpine - command: [sh] - source: | - apk add --no-cache jq curl - PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' - . as $root | - { - id: .after, - createdDate: (.commits | first | .timestamp), - startedDate: (.commits | first | .timestamp), - finishedDate: (.commits | last | .timestamp), - environment: "PRODUCTION", - result: "SUCCESS", - name: ("deployment-" + (.after | .[0:7])), - deploymentCommits: [ - .commits[] | { - repoUrl: $root.repository.clone_url, - refName: $root.ref, - startedDate: .timestamp, - finishedDate: .timestamp, - commitSha: .id, - commitMsg: .message, - result: "SUCCESS", - name: ("deployment-" + (.id | .[0:7])) - } - ] - }') - echo "====Transformed Payload====" - echo "$PAYLOAD" | jq '.' - TOKEN='{{inputs.parameters.token}}' - - curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' - parameters: - - src: - dependencyName: deployments-webhook - dataKey: body - dest: spec.arguments.parameters.0.value - diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data-linux.sh b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data-linux.sh new file mode 100644 index 000000000..cb3c79c32 --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data-linux.sh @@ -0,0 +1,185 @@ +#!/bin/sh + +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo "Example: $0 http://localhost:4000 your-api-key 2" + exit 1 +fi + +BASE_URL="$1" +API_KEY="$2" +CONNECTION_ID="$3" + + +rand() { + od -N 4 -t uL -An /dev/urandom | tr -d " " +} + +# Portable date functions +get_start_ts() { + date -d "180 days ago" +%s +} + +date_format() { + date -d "@$1" "+%Y-%m-%dT%H:%M:%S%z" +} + +create_pr() { + local pr_id=$1 + local created_date=$2 + local merged_date=$3 + local merge_commit_sha=$4 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/pull_requests" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{ + "id": "PR-'"$pr_id"'", + "headRepoId": "repo-'"$(($(rand) % 1000))"'", + "status": "MERGED", + "originalStatus": "OPEN", + "displayTitle": "Feature: Add new functionality '"$pr_id"'", + "description": "This PR adds new features", + "url": "https://github.com/org/repo/pull/'"$pr_id"'", + "pullRequestKey": '"$pr_id"', + "createdDate": "'"$created_date"'", + "mergedDate": "'"$merged_date"'", + "closedDate": null, + "mergeCommitSha": "'"$merge_commit_sha"'", + "headRef": "feature-branch-'"$pr_id"'", + "baseRef": "main", + "baseCommitSha": "'"$(openssl rand -hex 20)"'", + "headCommitSha": "'"$(openssl rand -hex 20)"'", + "isDraft": false + }') +} + +create_deployment() { + local deploy_id=$1 + local start_date=$2 + local commit_sha=$3 + local finish_date=$4 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/deployments" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"id\": \"DEPLOY-$deploy_id\", + \"startedDate\": \"$start_date\", + \"finishedDate\": \"$finish_date\", + \"result\": \"SUCCESS\", + \"url\": \"https://deploy.example.com/$deploy_id\", + \"deploymentCommits\":[ + { + \"repoUrl\": \"https://github.com/org/repo\", + \"refName\": \"main\", + \"startedDate\": \"$start_date\", + \"finishedDate\": \"$finish_date\", + \"commitSha\": \"$commit_sha\", + \"commitMsg\": \"Deployment $deploy_id\" + } + ] + }") + + echo "$finish_date" +} + +create_incident() { + local incident_id=$1 + local created_date=$2 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/issues" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"issueKey\":\"INC-$incident_id\", + \"title\":\"Incident $incident_id\", + \"type\":\"INCIDENT\", + \"originalStatus\":\"TODO\", + \"status\":\"TODO\", + \"createdDate\":\"$created_date\", + \"updatedDate\":\"$created_date\" + }") +} + +update_incident_status() { + local incident_id=$1 + local created_date=$2 + local updated_date=$3 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/issues" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"issueKey\":\"INC-$incident_id\", + \"title\":\"Incident $incident_id\", + \"type\":\"INCIDENT\", + \"originalStatus\":\"TODO\", + \"status\":\"DONE\", + \"createdDate\":\"$created_date\", + \"updatedDate\":\"$updated_date\", + \"resolutionDate\":\"$updated_date\" + }") +} + +cap_timestamp() { + local ts=$1 + local current_ts=$(date +%s) + if [ "$ts" -gt "$current_ts" ]; then + echo "$current_ts" + else + echo "$ts" + fi +} + +ONE_HOUR=3600 +ONE_DAY=$((24*ONE_HOUR)) +ONE_WEEK=$((ONE_DAY * 7)) + +end_ts=$(date +%s) +start_ts=$(get_start_ts) +current_ts=$start_ts +pr_id=1 +deploy_id=1 +incident_id=1 + +# Main loop to generate data for the last six months +while [ "$current_ts" -lt "$end_ts" ]; do + created_date=$(date_format "$current_ts") + + merged_ts=$(cap_timestamp $((current_ts + ONE_HOUR + $(rand) % ONE_WEEK))) + merged_date=$(date_format "$merged_ts") + merge_commit_sha=$(openssl rand -hex 20) + + create_pr "$pr_id" "$created_date" "$merged_date" "$merge_commit_sha" + + deploy_finish_ts=$(cap_timestamp $((merged_ts + ONE_HOUR + $(rand) % (2*ONE_HOUR)))) + deploy_finish_date=$(date_format "$deploy_finish_ts") + create_deployment "$deploy_id" "$merged_date" "$merge_commit_sha" "$deploy_finish_date" + + current_ts=$deploy_finish_ts + + if [ $(($(rand) % 10)) -eq 0 ]; then + incident_ts=$(cap_timestamp $((deploy_finish_ts + $(rand) % ONE_DAY))) + incident_date=$(date_format "$incident_ts") + create_incident "$incident_id" "$incident_date" + + fix_pr_merge_ts=$(cap_timestamp $((incident_ts + ONE_HOUR + $(rand) % (3*ONE_DAY)))) + fix_pr_merge_date=$(date_format "$fix_pr_merge_ts") + fix_merge_commit_sha=$(openssl rand -hex 20) + pr_id=$((pr_id+1)) + create_pr "$pr_id" "$incident_date" "$fix_pr_merge_date" "$fix_merge_commit_sha" + + fix_deploy_finish_ts=$(cap_timestamp $((fix_pr_merge_ts + ONE_HOUR + $(rand) % (2*ONE_HOUR)))) + fix_deploy_finish_date=$(date_format "$fix_deploy_finish_ts") + deploy_id=$((deploy_id+1)) + create_deployment "$deploy_id" "$fix_pr_merge_date" "$fix_merge_commit_sha" "$fix_deploy_finish_date" + + update_incident_status "$incident_id" "$incident_date" "$fix_deploy_finish_date" + incident_id=$((incident_id+1)) + + current_ts=$fix_deploy_finish_ts + fi + current_ts=$((current_ts + $(rand) % 2*ONE_DAY)) + pr_id=$((pr_id + 1)) + deploy_id=$((deploy_id + 1)) +done + +echo "Data generation complete. Created $((pr_id - 1)) pull requests, $((deploy_id - 1)) deployments, and $((incident_id - 1)) incidents." diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data.sh b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data.sh new file mode 100755 index 000000000..788a4293e --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data.sh @@ -0,0 +1,184 @@ +#!/bin/bash + +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo "Example: $0 http://localhost:4000 your-api-key 2" + exit 1 +fi + +BASE_URL="$1" +API_KEY="$2" +CONNECTION_ID="$3" + +rand() { + od -N 4 -t uL -An /dev/urandom | tr -d " " +} + +# Function to create a pull request and return the merge commit SHA +create_pr() { + local pr_id=$1 + local created_date=$2 + local merged_date=$3 + local merge_commit_sha=$4 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/pull_requests" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{ + "id": "PR-'"$pr_id"'", + "headRepoId": "repo-'"$(($(rand) % 1000))"'", + "status": "MERGED", + "originalStatus": "OPEN", + "displayTitle": "Feature: Add new functionality '"$pr_id"'", + "description": "This PR adds new features", + "url": "https://github.com/org/repo/pull/'"$pr_id"'", + "pullRequestKey": '"$pr_id"', + "createdDate": "'"$created_date"'", + "mergedDate": "'"$merged_date"'", + "closedDate": null, + "mergeCommitSha": "'"$merge_commit_sha"'", + "headRef": "feature-branch-'"$pr_id"'", + "baseRef": "main", + "baseCommitSha": "'"$(openssl rand -hex 20)"'", + "headCommitSha": "'"$(openssl rand -hex 20)"'", + "isDraft": false + }') +} + +# Function to create a deployment +create_deployment() { + local deploy_id=$1 + local start_date=$2 + local commit_sha=$3 + local finish_date=$4 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/deployments" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"id\": \"DEPLOY-$deploy_id\", + \"startedDate\": \"$start_date\", + \"finishedDate\": \"$finish_date\", + \"result\": \"SUCCESS\", + \"url\": \"https://deploy.example.com/$deploy_id\", + \"deploymentCommits\":[ + { + \"repoUrl\": \"https://github.com/org/repo\", + \"refName\": \"main\", + \"startedDate\": \"$start_date\", + \"finishedDate\": \"$finish_date\", + \"commitSha\": \"$commit_sha\", + \"commitMsg\": \"Deployment $deploy_id\" + } + ] + }") + + echo $finish_date +} + +# Function to create an incident +create_incident() { + local incident_id=$1 + local created_date=$2 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/issues" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"issueKey\":\"INC-$incident_id\", + \"title\":\"Incident $incident_id\", + \"type\":\"INCIDENT\", + \"originalStatus\":\"TODO\", + \"status\":\"TODO\", + \"createdDate\":\"$created_date\", + \"updatedDate\":\"$created_date\" + }") +} + +# Function to update incident status to DONE +update_incident_status() { + local incident_id=$1 + local created_date=$2 + local updated_date=$3 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/issues" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"issueKey\":\"INC-$incident_id\", + \"title\":\"Incident $incident_id\", + \"type\":\"INCIDENT\", + \"originalStatus\":\"TODO\", + \"status\":\"DONE\", + \"createdDate\":\"$created_date\", + \"updatedDate\":\"$updated_date\", + \"resolutionDate\":\"$updated_date\" + }") +} + +cap_timestamp() { + local ts=$1 + local current_ts=$(date +%s) + if [ $ts -gt $current_ts ]; then + echo $current_ts + else + echo $ts + fi +} + +ONE_HOUR=3600 +ONE_DAY=$((24*ONE_HOUR)) +ONE_WEEK=$((ONE_DAY * 7)) + +end_ts=$(date +%s) +start_ts=$(date -v-180d +%s) +current_ts=$start_ts +pr_id=1 +deploy_id=1 +incident_id=1 + +# Main loop to generate data for the last six months +while [ $current_ts -lt $end_ts ];do + # Generate random timestamp more efficiently + created_date=$(date -r $current_ts +"%Y-%m-%dT%H:%M:%S%z") + + # Create PR + merged_ts=$(cap_timestamp $((current_ts + ONE_HOUR + $(rand) % ONE_WEEK))) + merged_date=$(date -r $merged_ts +"%Y-%m-%dT%H:%M:%S%z") + merge_commit_sha=$(openssl rand -hex 20) + + create_pr $pr_id "$created_date" "$merged_date" "$merge_commit_sha" + + # Create deployment + deploy_finish_ts=$(cap_timestamp $((merged_ts + ONE_HOUR + $(rand) % (2*ONE_HOUR)))) + deploy_finish_date=$(date -r $deploy_finish_ts +"%Y-%m-%dT%H:%M:%S%z") + create_deployment $deploy_id "$merged_date" "$merge_commit_sha" "$deploy_finish_date" + + current_ts=$deploy_finish_ts + # 1/7 chance of creating an incident + if [ $(($(rand) % 10)) -eq 0 ]; then + incident_ts=$(cap_timestamp $((deploy_finish_ts + $(rand) % ONE_DAY))) + incident_date=$(date -r $incident_ts +"%Y-%m-%dT%H:%M:%S%z") + create_incident $incident_id "$incident_date" + + # Create fix PR and deployment + fix_pr_merge_ts=$(cap_timestamp $((incident_ts + ONE_HOUR + $(rand) % (3*ONE_DAY)))) + fix_pr_merge_date=$(date -r $fix_pr_merge_ts +"%Y-%m-%dT%H:%M:%S%z") + fix_merge_commit_sha=$(openssl rand -hex 20) + pr_id=$((pr_id+1)) + create_pr $pr_id "$incident_date" "$fix_pr_merge_date" "$fix_merge_commit_sha" + + fix_deploy_finish_ts=$(cap_timestamp $((fix_pr_merge_ts + ONE_HOUR + $(rand) % (2*ONE_HOUR)))) + fix_deploy_finish_date=$(date -r $fix_deploy_finish_ts +"%Y-%m-%dT%H:%M:%S%z") + deploy_id=$((deploy_id+1)) + create_deployment $deploy_id "$fix_pr_merge_date" "$fix_merge_commit_sha" "$fix_deploy_finish_date" + + update_incident_status $incident_id "$incident_date" "$fix_deploy_finish_date" + incident_id=$((incident_id+1)) + + current_ts=$fix_deploy_finish_ts + fi + current_ts=$((current_ts + $(rand) % 2*ONE_DAY)) + pr_id=$((pr_id + 1)) + deploy_id=$((deploy_id + 1)) +done + +echo "Data generation complete. Created $((pr_id - 1)) pull requests, $((deploy_id - 1)) deployments, and $((incident_id - 1)) incidents." + diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/init.sh b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/init.sh new file mode 100644 index 000000000..bc224cca2 --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/init.sh @@ -0,0 +1,83 @@ +#!/bin/bash +if [ $# -lt 2 ]; then + echo "Usage: $0 " + echo "Example: $0 http://localhost:4000 rust" + exit 1 +fi + +BASE_URL=$1 +curl -X GET ${BASE_URL}/proceed-db-migration + +projectName=$2 + +projectCreateRequest=$( + cat <.*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" + sed -i 's+image: '"$image"'.*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" + fi + m_count=$(git status -s | wc -l) + if [ $m_count -ge 1 ]; then + git add "{{inputs.parameters.deployment-path}}" + image_version="{{inputs.parameters.tag}}" + git commit -m "Add Exit Handler for DORA Metrics" + git push --set-upstream origin main + else + echo "No updates to values file, skipping git push" + fi + +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: devlake-data-init-template + namespace: ${{values.namespace}} +spec: + arguments: + parameters: + - name: appname + value: ${{values.appname}} + - name: apprepo + value: ${{values.hostname}}/gitea/giteaAdmin/${{values.appname}}-cicd + + templates: + - name: devlake-init + dag: + tasks: + - name: clean + template: cleandir + + - name: clone + template: clonerepo + arguments: + parameters: + - name: apprepo + value: "{{workflow.parameters.apprepo}}" + - name: branch + value: main + depends: "clean" + + - name: warmup + template: build-image-for-cache + arguments: + parameters: + - name: dockerfile-path + value: "{{workflow.parameters.dockerfile_path}}" + - name: tag-version + value: init + depends: "clone" + + - name: edit-tag + template: update-image-tag + arguments: + parameters: + - name: env + value: dev + - name: tag + value: init + - name: apprepo + value: "{{workflow.parameters.apprepo}}" + - name: deployment-path + value: "{{workflow.parameters.deployment_path}}" + depends: "warmup" + + - name: cleandir + container: + volumeMounts: + - mountPath: /workdir + name: workdir + image: alpine:3.20 + workingDir: /workdir + args: + - /bin/sh + - -c + - | + set -ex + if [ "$(ls -A /workdir)" ]; then + echo "Directory /workdir is not empty. Removing contents..." + rm -rf /workdir/* + fi + + - name: clonerepo + inputs: + parameters: + - name: apprepo + - name: branch + container: + volumeMounts: + - mountPath: /workdir + name: workdir + image: alpine/git:v2.45.1 + workingDir: /workdir + args: + - clone + - --depth + - "1" + - --branch + - "{{inputs.parameters.branch}}" + - --single-branch + - "https://{{inputs.parameters.apprepo}}" + - -c + - http.sslVerify=false + - . + + - name: build-image-for-cache + inputs: + parameters: + - name: tag-version + - name: dockerfile-path + volumes: + - name: docker-config + secret: + secretName: "{{workflow.parameters.aws_region}}-regcred" + container: + image: gcr.io/kaniko-project/executor:debug + volumeMounts: + - name: workdir + mountPath: /workdir + - name: docker-config + mountPath: /kaniko/.docker + workingDir: /workdir + resources: + limits: + cpu: 2 + memory: 5Gi + env: + - name: DOCKER_CONFIG + value: /kaniko/.docker + - name: ECR_REPO + valueFrom: + configMapKeyRef: + name: "{{workflow.parameters.appname}}-ecr-repo" + key: image + command: [ + "/busybox/sh", + "-c", + "/kaniko/executor \ + --dockerfile=Dockerfile \ + --context={{inputs.parameters.dockerfile-path}} \ + --destination=${ECR_REPO}:{{inputs.parameters.tag-version}} \ + --cache=true \ + --skip-tls-verify" + ] + + - name: update-image-tag + inputs: + parameters: + - name: env + - name: tag + - name: apprepo + - name: deployment-path + container: + envFrom: + - secretRef: + name: gitea-credentials + - configMapRef: + name: "{{workflow.parameters.appname}}-ecr-repo" + volumeMounts: + - mountPath: /workdir + name: workdir + image: alpine/git:v2.45.2 + workingDir: /workdir + command: ['/bin/sh'] + args: + - -c + - | + apk add --no-cache sed + set -ex + DEPLOY_PATH="{{inputs.parameters.deployment-path}}/{{inputs.parameters.env}}" + tag="{{inputs.parameters.tag}}" + mkdir -p $DEPLOY_PATH + git config --global user.email "noreply@aws.com" + git config --global user.name ARGO + git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git + git pull --rebase + echo "$image:{{inputs.parameters.tag}}" > ./deployment/"{{inputs.parameters.env}}"/image.txt + if [ -f $DEPLOY_PATH/application.yml ]; then + sed -i 's+image: .*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" + sed -i 's+image: '"$image"'.*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" + fi + m_count=$(git status -s | wc -l) + if [ $m_count -ge 1 ]; then + git add "{{inputs.parameters.deployment-path}}" + image_version="{{inputs.parameters.tag}}" + git commit -m "[SKIP CI] Updated image version to $image_version" + git push --set-upstream origin main + else + echo "No updates to values file, skipping git push" + fi + + + + diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/catalog-info.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/catalog-info.yaml new file mode 100644 index 000000000..b54ccd247 --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/catalog-info.yaml @@ -0,0 +1,22 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: ${{values.appname}}-devlake + description: Onboard ${{ values.appname }} team to DevLake + annotations: + backstage.io/kubernetes-namespace: ${{values.namespace}} + backstage.io/kubernetes-label-selector: 'entity-id=${{values.appname}}-devlake' + argocd/app-selector: 'entity-id=${{values.appname}}-devlake' + argo-workflows.cnoe.io/cluster-name: local + argo-workflows.cnoe.io/namespace: team-${{values.appname}} + argo-workflows.cnoe.io/label-selector: 'entity-id=${{values.appname}}-devlake' + links: + - url: https://${{values.hostname}}/gitea?repo=${{values.appname}}-devlake + title: Repo URL + icon: github +spec: + type: service + lifecycle: experimental + owner: guest + system: ${{values.appname | dump}} + diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/bus.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/bus.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml new file mode 100644 index 000000000..b7f47a93b --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml @@ -0,0 +1,152 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitea-devlake-webhook-svc +spec: + type: ClusterIP # Or ClusterIP if using Ingress + ports: + - port: 12000 + targetPort: 12000 + protocol: TCP + authSecret: + name: + selector: + eventsource-name: gitea-devlake-webhook + +--- +apiVersion: argoproj.io/v1alpha1 +kind: EventBus +metadata: + name: default +spec: + nats: + native: + replicas: 3 + auth: token +--- +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: gitea-devlake-webhook + labels: + eventsource-name: gitea-devlake-webhook +spec: + service: + ports: + - port: 12000 + targetPort: 12000 + webhook: + incidents-webhook: + port: "12000" + endpoint: /webhook/incidents + method: POST + deployments-webhook: + port: "12000" + endpoint: /webhook/deployments + method: POST + pull-requests-webhook: + port: "12000" + endpoint: "/webhook/pull-requests" + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: gitea-webhook-sensor +spec: + template: + serviceAccountName: wf-sa + dependencies: + - name: incidents-webhook + eventSourceName: gitea-webhook + eventName: incidents-webhook + - name: deployments-webhook + eventSourceName: gitea-webhook + eventName: deployments-webhook + - name: pull-requests-webhook + eventSourceName: gitea-webhook + eventName: pull-requests-webhook + triggers: + - template: + name: incidents-workflow-trigger + conditions: "incidents-webhook" + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: incidents-workflow- + spec: + workflowTemplateRef: + name: incidents-workflow-template + arguments: + parameters: + - name: body + value: '{{.Input.body}}' + - name: headers + value: '{{.Input.headers}}' + - name: url + value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/issues + parameters: + - src: + dependencyName: incidents-webhook + dataKey: body + dest: spec.arguments.parameters.0.value + + - template: + name: deployments-workflow-trigger + conditions: "deployments-webhook" + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: deployments-workflow- + spec: + workflowTemplateRef: + name: deployments-workflow-template + arguments: + parameters: + - name: body + value: '{{.Input.body}}' + - name: headers + value: '{{.Input.headers}}' + - name: url + value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/deployments + parameters: + - src: + dependencyName: deployments-webhook + dataKey: body + dest: spec.arguments.parameters.0.value + + - template: + name: pull-requests-workflow-trigger + conditions: "pull-requests-webhook" + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: pull-requests-workflow- + spec: + workflowTemplateRef: + name: pull-requests-workflow-template + arguments: + parameters: + - name: body + value: '{{.Input.body}}' + - name: headers + value: '{{.Input.headers}}' + - name: url + value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/pullrequests + parameters: + - src: + dependencyName: pull-requests-webhook + dataKey: body + dest: spec.arguments.parameters.0.value diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml new file mode 100644 index 000000000..c8f5e7c80 --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml @@ -0,0 +1,25 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: gitea-credentials + namespace: ${{values.namespace}} +spec: + secretStoreRef: + name: gitea + kind: ClusterSecretStore + refreshInterval: "0" + target: + name: gitea-credentials + data: + - secretKey: GITEA_USERNAME + remoteRef: + key: gitea-credential + property: username + - secretKey: GITEA_PASSWORD + remoteRef: + key: gitea-credential + property: password + - secretKey: GITEA_TOKEN + remoteRef: + key: gitea-credential + property: token \ No newline at end of file diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml new file mode 100644 index 000000000..d1ed3409f --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: argo-rollouts-notification-configmap + namespace: ${{values.namespace}} +data: + trigger.on-rollout-aborted: | + - send: [modengg-rollout-aborted] + trigger.on-rollout-completed: | + - send: [modengg-rollout-completed] + service.webhook.deployments: | + url: https://${{values.hostname}}/argo-events/deployments + template.modengg-rollout-aborted: | + - stuff + template.modengg-rollout-completed: | + - stuff + + diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml new file mode 100644 index 000000000..9d18bc9dd --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml @@ -0,0 +1,175 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: incidents-workflow-template +spec: + entrypoint: process-incident + serviceAccountName: devlake-webhook-sa + arguments: + parameters: + - name: body + - name: headers + - name: url + templates: + - name: process-incident + inputs: + parameters: + - name: body + - name: headers + - name: url + script: + image: alpine + command: [sh] + env: + - name: DEVLAKE_TOKEN + valueFrom: + secretKeyRef: + name: devlake-webhook-secret + key: token + source: | + apk add --no-cache jq curl + echo '{{inputs.parameters.body}}' + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + { + issueKey: (.repository.full_name + "-" + (.issue.number | tostring)), + title: .issue.title, + description: .issue.body, + url: .issue.html_url, + type: "INCIDENT", + status: (if .issue.state == "open" then "TODO" else "DONE" end), + originalStatus: "TODO", + createdDate: .issue.created_at, + updatedDate: .issue.updated_at, + resolutionDate: (if .issue.state == "closed" then .issue.closed_at else null end), + priority: "", + severity: "", + creatorId: .issue.user.login, + creatorName: (.issue.user.full_name // .issue.user.login), + assigneeId: (.issue.assignee.login // ""), + assigneeName: (if .issue.assignee then (.issue.assignee.full_name // .issue.assignee.login) else "" end) + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: deployments-workflow-template +spec: + entrypoint: process-deployment + serviceAccountName: devlake-webhook-sa + arguments: + parameters: + - name: body + - name: headers + - name: url + templates: + - name: process-deployment + inputs: + parameters: + - name: body + - name: headers + - name: url + script: + image: alpine + command: [sh] + env: + - name: DEVLAKE_TOKEN + valueFrom: + secretKeyRef: + name: devlake-webhook-secret + key: token + source: | + apk add --no-cache jq curl + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + . as $root | + { + id: .after, + createdDate: (.commits | first | .timestamp), + startedDate: (.commits | first | .timestamp), + finishedDate: (.commits | last | .timestamp), + environment: "PRODUCTION", + result: "SUCCESS", + name: ("deployment-" + (.after | .[0:7])), + deploymentCommits: [ + .commits[] | { + repoUrl: $root.repository.clone_url, + refName: $root.ref, + startedDate: .timestamp, + finishedDate: .timestamp, + commitSha: .id, + commitMsg: .message, + result: "SUCCESS", + name: ("deployment-" + (.id | .[0:7])) + } + ] + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pull-requests-workflow-template +spec: + entrypoint: process-pull-request + serviceAccountName: devlake-webhook-sa + arguments: + parameters: + - name: body + - name: headers + - name: url + templates: + - name: process-pull-request + inputs: + parameters: + - name: body + - name: headers + - name: url + script: + image: alpine + command: [sh] + env: + - name: DEVLAKE_TOKEN + valueFrom: + secretKeyRef: + name: devlake-webhook-secret + key: token + + source: | + apk add --no-cache jq curl + echo '{{inputs.parameters.body}}' + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + { + id: (.pull_request.id | tostring), + title: .pull_request.title, + description: .pull_request.body, + status: (if .pull_request.state == "open" then "OPEN" elif .pull_request.merged then "MERGED" else "CLOSED" end), + createdDate: .pull_request.created_at, + updatedDate: .pull_request.updated_at, + closedDate: .pull_request.closed_at, + mergedDate: .pull_request.merged_at, + type: "PR", + url: .pull_request.html_url, + authorId: .pull_request.user.login, + authorName: (.pull_request.user.name // .pull_request.user.login), + reviewers: [.pull_request.requested_reviewers[].login], + comments: (.pull_request.comments // 0), + commits: (.pull_request.commits // 0), + additions: (.pull_request.additions // 0), + deletions: (.pull_request.deletions // 0), + changedFiles: (.pull_request.changed_files // 0), + baseBranch: .pull_request.base.ref, + headBranch: .pull_request.head.ref, + repoId: .repository.id, + repoName: .repository.full_name + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 565426073..8df18c158 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -2,7 +2,7 @@ resource "aws_iam_policy" "external-secrets-devlake" { count = local.secret_count name_prefix = "modern-engg-external-secrets-devlake-" - description = "For use with External Secrets Controller for Devlake" + description = "For use with External Secrets Controller for DevLake" policy = jsonencode( { "Version" : "2012-10-17", From 98acf2bad72250d65d7daf1cb68c4f0109f5aa22 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 2 Apr 2025 17:00:54 -0700 Subject: [PATCH 20/63] updates to backstage templates --- .../provisioner/wf-run.yaml | 14 +- .../provisioner/wf-templates.yaml | 520 +++++++++++------- .../skeleton/manifests/bus.yaml | 0 .../skeleton/manifests/events.yaml | 22 +- .../skeleton/manifests/exit-handler.yaml | 120 ++++ .../manifests/gitea-external-secret.yaml | 25 - .../skeleton/manifests/rbac.yaml | 19 + .../skeleton/manifests/rollout-notifs.yaml | 18 - .../skeleton/manifests/wf-templates.yaml | 36 +- 9 files changed, 479 insertions(+), 295 deletions(-) delete mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/bus.yaml delete mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml create mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rbac.yaml delete mode 100644 platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml index df7a990d3..af445f29d 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml @@ -42,16 +42,8 @@ spec: value: "{{workflow.parameters.appname}}" - name: namespace value: "{{workflow.parameters.namespace}}" - - name: hostname - value: "{{workflow.parameters.hostname}}" - - - name: run-init + - - name: run-data-gen templateRef: - name: devlake-data-init-template - template: devlake-init - arguments: - parameters: - - name: appname - value: "{{workflow.parameters.appname}}" - - name: apprepo - value: "{{workflow.parameters.apprepo}}" + name: devlake-data-gen-template + template: devlake-data-gen diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml index 50afd5e79..1b09e9798 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml @@ -17,33 +17,33 @@ spec: - name: devlake-provisioner dag: tasks: - - name: create-devlake-webhook-secret - template: create-devlake-webhook-secret - - - name: get-ecr-secret - template: get-ecr-secret - - - name: create-docker-registry-secret - template: create-docker-registry-secret + - name: devlake-init + template: devlake-init arguments: parameters: - - name: ecr_secret - value: "{{tasks.get-ecr-secret.outputs.result}}" - depends: "get-ecr-secret" - - - name: create-ecr-repository - template: create-ecr-repository + - name: appname + value: "{{workflow.parameters.appname}}" + - name: url + value: "{{workflow.parameters.url}}" - - name: create-ecr-configmap - template: create-ecr-configmap + - name: create-devlake-webhook-secret-and-cm + template: create-devlake-webhook-secret-and-cm arguments: parameters: - - name: image - value: "{{tasks.create-ecr-repository.outputs.result}}" - depends: "create-ecr-repository" + - name: token-and-id + value: "{{tasks.devlake-init.outputs.result}}" + depends: "devlake-init" - name: create-gitea-webhook template: create-gitea-webhook + depends: "create-devlake-webhook-secret-and-cm" + + - name: update-cicd-exit-hook + template: update-cicd-exit-hook + arguments: + parameters: + - name: apprepo + value: "{{workflow.parameters.appname}}-cicd" - name: devlake-init inputs: @@ -55,20 +55,99 @@ spec: command: ["sh"] args: - | - apk install curl + apk install curl jq + BASE_URL="{{inputs.parameters.url}}" + curl -X GET ${BASE_URL}/proceed-db-migration + + projectName="{{inputs.parameters.appname}}" + + projectCreateRequest=$( + cat < 0') + if ! $webhook_exists; then echo "Webhook is not found, creating webhook with $webhook_url" - curl -k -X 'POST' "${hooks_api_url}" -H "accept: application/json" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" -d '{"branch_filter": "main","type": "gitea", "events":["push"], "config": {"url": "'${webhook_url}'", "content_type": "json"}, "active": true}' + curl -k -X 'POST' "${hooks_api_url}" -H "accept: application/json" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" -d '{"branch_filter": "*","type": "gitea", "events":["issues"], "config": {"url": "'${webhook_url}'", "content_type": "json"}, "active": true}' else - echo "Webhook is already exists" + echo "Webhook with url $webhook_url already exists" + fi + + webhook_url="http://{{workflow.parameters.url}}"/argo-events/"{{workflow.parameters.appname}}-dora/pull-requests" + webhook_exists=$(curl -k -X 'GET' "${hooks_api_url}" -H "accept: application/json" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" | jq --arg url "$webhook_url" 'map(select(.config.url == $url)) | length > 0') + if ! $webhook_exists; then + echo "Webhook is not found, creating webhook with $webhook_url" + curl -k -X 'POST' "${hooks_api_url}" -H "accept: application/json" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" -d '{"branch_filter": "main","type": "gitea", "events":["pull_request"], "config": {"url": "'${webhook_url}'", "content_type": "json"}, "active": true}' + else + echo "Webhook with url $webhook_url already exists" fi - name: update-cicd-exit-hook inputs: parameters: - name: apprepo + - name: exit-hook-name container: envFrom: - secretRef: @@ -108,207 +197,226 @@ spec: args: - -c - | - apk add --no-cache sed + apk add --no-cache yq set -ex git config --global user.email "noreply@aws.com" git config --global user.name ARGO git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git git pull --rebase - if [ -f $DEPLOY_PATH/application.yml ]; then - sed -i 's+image: .*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" - sed -i 's+image: '"$image"'.*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" + if [ -f manifests/cicd-pipeline.yml ]; then + yq 'select(.kind == "Sensor").spec.triggers[].template.argoWorkflow.source.resource.spec.onExit = "{{inputs.parameters.exit-hook-name}}"' -i manifests/cicd-pipeline.yaml fi m_count=$(git status -s | wc -l) if [ $m_count -ge 1 ]; then - git add "{{inputs.parameters.deployment-path}}" - image_version="{{inputs.parameters.tag}}" + git add . git commit -m "Add Exit Handler for DORA Metrics" git push --set-upstream origin main else - echo "No updates to values file, skipping git push" + echo "No updates, skipping git push" fi --- apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: devlake-data-init-template + name: devlake-data-gen-template namespace: ${{values.namespace}} spec: - arguments: - parameters: - - name: appname - value: ${{values.appname}} - - name: apprepo - value: ${{values.hostname}}/gitea/giteaAdmin/${{values.appname}}-cicd - templates: - - name: devlake-init + - name: devlake-data-gen dag: tasks: - - name: clean - template: cleandir - - - name: clone - template: clonerepo - arguments: - parameters: - - name: apprepo - value: "{{workflow.parameters.apprepo}}" - - name: branch - value: main - depends: "clean" - - - name: warmup - template: build-image-for-cache - arguments: - parameters: - - name: dockerfile-path - value: "{{workflow.parameters.dockerfile_path}}" - - name: tag-version - value: init - depends: "clone" + - name: data-gen + template: devlake-generate-data - - name: edit-tag - template: update-image-tag - arguments: - parameters: - - name: env - value: dev - - name: tag - value: init - - name: apprepo - value: "{{workflow.parameters.apprepo}}" - - name: deployment-path - value: "{{workflow.parameters.deployment_path}}" - depends: "warmup" - - - name: cleandir - container: - volumeMounts: - - mountPath: /workdir - name: workdir - image: alpine:3.20 - workingDir: /workdir - args: - - /bin/sh - - -c - - | - set -ex - if [ "$(ls -A /workdir)" ]; then - echo "Directory /workdir is not empty. Removing contents..." - rm -rf /workdir/* - fi - - - name: clonerepo - inputs: - parameters: - - name: apprepo - - name: branch - container: - volumeMounts: - - mountPath: /workdir - name: workdir - image: alpine/git:v2.45.1 - workingDir: /workdir - args: - - clone - - --depth - - "1" - - --branch - - "{{inputs.parameters.branch}}" - - --single-branch - - "https://{{inputs.parameters.apprepo}}" - - -c - - http.sslVerify=false - - . - - - name: build-image-for-cache - inputs: - parameters: - - name: tag-version - - name: dockerfile-path - volumes: - - name: docker-config - secret: - secretName: "{{workflow.parameters.aws_region}}-regcred" - container: - image: gcr.io/kaniko-project/executor:debug - volumeMounts: - - name: workdir - mountPath: /workdir - - name: docker-config - mountPath: /kaniko/.docker - workingDir: /workdir - resources: - limits: - cpu: 2 - memory: 5Gi - env: - - name: DOCKER_CONFIG - value: /kaniko/.docker - - name: ECR_REPO - valueFrom: - configMapKeyRef: - name: "{{workflow.parameters.appname}}-ecr-repo" - key: image - command: [ - "/busybox/sh", - "-c", - "/kaniko/executor \ - --dockerfile=Dockerfile \ - --context={{inputs.parameters.dockerfile-path}} \ - --destination=${ECR_REPO}:{{inputs.parameters.tag-version}} \ - --cache=true \ - --skip-tls-verify" - ] - - - name: update-image-tag - inputs: - parameters: - - name: env - - name: tag - - name: apprepo - - name: deployment-path + - name: devlake-generate-data container: + image: debian:12 + command: [/bin/bash] envFrom: - - secretRef: - name: gitea-credentials - - configMapRef: - name: "{{workflow.parameters.appname}}-ecr-repo" - volumeMounts: - - mountPath: /workdir - name: workdir - image: alpine/git:v2.45.2 - workingDir: /workdir - command: ['/bin/sh'] + - secretRef: + name: devlake-webhook-secret + - configMapRef: + name: devlake-webhook-id args: - - -c - - | - apk add --no-cache sed - set -ex - DEPLOY_PATH="{{inputs.parameters.deployment-path}}/{{inputs.parameters.env}}" - tag="{{inputs.parameters.tag}}" - mkdir -p $DEPLOY_PATH - git config --global user.email "noreply@aws.com" - git config --global user.name ARGO - git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git - git pull --rebase - echo "$image:{{inputs.parameters.tag}}" > ./deployment/"{{inputs.parameters.env}}"/image.txt - if [ -f $DEPLOY_PATH/application.yml ]; then - sed -i 's+image: .*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" - sed -i 's+image: '"$image"'.*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" - fi - m_count=$(git status -s | wc -l) - if [ $m_count -ge 1 ]; then - git add "{{inputs.parameters.deployment-path}}" - image_version="{{inputs.parameters.tag}}" - git commit -m "[SKIP CI] Updated image version to $image_version" - git push --set-upstream origin main - else - echo "No updates to values file, skipping git push" - fi - - - + - | + apt update + apt install -y curl openssl coreutils + BASE_URL="http://devlake-ui.devlake.svc.cluster.local:4000" + API_KEY="$DEVLAKE_TOKEN" + CONNECTION_ID="$DEVLAKE_HOOK_ID" + + + rand() { + od -N 4 -t uL -An /dev/urandom | tr -d " " + } + + # Portable date functions + get_start_ts() { + date -d "180 days ago" +%s + } + + date_format() { + date -d "@$1" "+%Y-%m-%dT%H:%M:%S%z" + } + + create_pr() { + local pr_id=$1 + local created_date=$2 + local merged_date=$3 + local merge_commit_sha=$4 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/pull_requests" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{ + "id": "PR-'"$pr_id"'", + "headRepoId": "repo-'"$(($(rand) % 1000))"'", + "status": "MERGED", + "originalStatus": "OPEN", + "displayTitle": "Feature: Add new functionality '"$pr_id"'", + "description": "This PR adds new features", + "url": "https://github.com/org/repo/pull/'"$pr_id"'", + "pullRequestKey": '"$pr_id"', + "createdDate": "'"$created_date"'", + "mergedDate": "'"$merged_date"'", + "closedDate": null, + "mergeCommitSha": "'"$merge_commit_sha"'", + "headRef": "feature-branch-'"$pr_id"'", + "baseRef": "main", + "baseCommitSha": "'"$(openssl rand -hex 20)"'", + "headCommitSha": "'"$(openssl rand -hex 20)"'", + "isDraft": false + }') + } + + create_deployment() { + local deploy_id=$1 + local start_date=$2 + local commit_sha=$3 + local finish_date=$4 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/deployments" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"id\": \"DEPLOY-$deploy_id\", + \"startedDate\": \"$start_date\", + \"finishedDate\": \"$finish_date\", + \"result\": \"SUCCESS\", + \"url\": \"https://deploy.example.com/$deploy_id\", + \"deploymentCommits\":[ + { + \"repoUrl\": \"https://github.com/org/repo\", + \"refName\": \"main\", + \"startedDate\": \"$start_date\", + \"finishedDate\": \"$finish_date\", + \"commitSha\": \"$commit_sha\", + \"commitMsg\": \"Deployment $deploy_id\" + } + ] + }") + + echo "$finish_date" + } + + create_incident() { + local incident_id=$1 + local created_date=$2 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/issues" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"issueKey\":\"INC-$incident_id\", + \"title\":\"Incident $incident_id\", + \"type\":\"INCIDENT\", + \"originalStatus\":\"TODO\", + \"status\":\"TODO\", + \"createdDate\":\"$created_date\", + \"updatedDate\":\"$created_date\" + }") + } + + update_incident_status() { + local incident_id=$1 + local created_date=$2 + local updated_date=$3 + + response=$(curl -s "${BASE_URL}/api/rest/plugins/webhook/connections/${CONNECTION_ID}/issues" -X 'POST' \ + -H "Authorization: Bearer ${API_KEY}" \ + -d "{ + \"issueKey\":\"INC-$incident_id\", + \"title\":\"Incident $incident_id\", + \"type\":\"INCIDENT\", + \"originalStatus\":\"TODO\", + \"status\":\"DONE\", + \"createdDate\":\"$created_date\", + \"updatedDate\":\"$updated_date\", + \"resolutionDate\":\"$updated_date\" + }") + } + + cap_timestamp() { + local ts=$1 + local current_ts=$(date +%s) + if [ "$ts" -gt "$current_ts" ]; then + echo "$current_ts" + else + echo "$ts" + fi + } + + ONE_HOUR=3600 + ONE_DAY=$((24*ONE_HOUR)) + ONE_WEEK=$((ONE_DAY * 7)) + + end_ts=$(date +%s) + start_ts=$(get_start_ts) + current_ts=$start_ts + pr_id=1 + deploy_id=1 + incident_id=1 + + # Main loop to generate data for the last six months + while [ "$current_ts" -lt "$end_ts" ]; do + created_date=$(date_format "$current_ts") + + merged_ts=$(cap_timestamp $((current_ts + ONE_HOUR + $(rand) % ONE_WEEK))) + merged_date=$(date_format "$merged_ts") + merge_commit_sha=$(openssl rand -hex 20) + + create_pr "$pr_id" "$created_date" "$merged_date" "$merge_commit_sha" + + deploy_finish_ts=$(cap_timestamp $((merged_ts + ONE_HOUR + $(rand) % (2*ONE_HOUR)))) + deploy_finish_date=$(date_format "$deploy_finish_ts") + create_deployment "$deploy_id" "$merged_date" "$merge_commit_sha" "$deploy_finish_date" + + current_ts=$deploy_finish_ts + + if [ $(($(rand) % 10)) -eq 0 ]; then + incident_ts=$(cap_timestamp $((deploy_finish_ts + $(rand) % ONE_DAY))) + incident_date=$(date_format "$incident_ts") + create_incident "$incident_id" "$incident_date" + + fix_pr_merge_ts=$(cap_timestamp $((incident_ts + ONE_HOUR + $(rand) % (3*ONE_DAY)))) + fix_pr_merge_date=$(date_format "$fix_pr_merge_ts") + fix_merge_commit_sha=$(openssl rand -hex 20) + pr_id=$((pr_id+1)) + create_pr "$pr_id" "$incident_date" "$fix_pr_merge_date" "$fix_merge_commit_sha" + + fix_deploy_finish_ts=$(cap_timestamp $((fix_pr_merge_ts + ONE_HOUR + $(rand) % (2*ONE_HOUR)))) + fix_deploy_finish_date=$(date_format "$fix_deploy_finish_ts") + deploy_id=$((deploy_id+1)) + create_deployment "$deploy_id" "$fix_pr_merge_date" "$fix_merge_commit_sha" "$fix_deploy_finish_date" + + update_incident_status "$incident_id" "$incident_date" "$fix_deploy_finish_date" + incident_id=$((incident_id+1)) + + current_ts=$fix_deploy_finish_ts + fi + current_ts=$((current_ts + $(rand) % 2*ONE_DAY)) + pr_id=$((pr_id + 1)) + deploy_id=$((deploy_id + 1)) + done + + echo "Data generation complete. Created $((pr_id - 1)) pull requests, $((deploy_id - 1)) deployments, and $((incident_id - 1)) incidents." diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/bus.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/bus.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml index b7f47a93b..4a19c1128 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml @@ -1,30 +1,20 @@ apiVersion: v1 kind: Service metadata: - name: gitea-devlake-webhook-svc + name: devlake-webhook-svc spec: type: ClusterIP # Or ClusterIP if using Ingress ports: - port: 12000 targetPort: 12000 protocol: TCP - authSecret: - name: + #authSecret: + #name: selector: eventsource-name: gitea-devlake-webhook --- apiVersion: argoproj.io/v1alpha1 -kind: EventBus -metadata: - name: default -spec: - nats: - native: - replicas: 3 - auth: token ---- -apiVersion: argoproj.io/v1alpha1 kind: EventSource metadata: name: gitea-devlake-webhook @@ -38,15 +28,15 @@ spec: webhook: incidents-webhook: port: "12000" - endpoint: /webhook/incidents + endpoint: /incidents method: POST deployments-webhook: port: "12000" - endpoint: /webhook/deployments + endpoint: /deployments method: POST pull-requests-webhook: port: "12000" - endpoint: "/webhook/pull-requests" + endpoint: "/pull-requests" --- apiVersion: argoproj.io/v1alpha1 diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml index e69de29bb..c7632e84b 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml @@ -0,0 +1,120 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: cicd-deployments-exit-handler +spec: + entrypoint: handle-exit + serviceAccountName: devlake-webhook-sa + arguments: + parameters: + - name: body + - name: headers + - name: url + templates: + - name: process-deployment + inputs: + parameters: + - name: body + - name: headers + - name: url + script: + image: alpine + command: [sh] + env: + - name: DEVLAKE_TOKEN + valueFrom: + secretKeyRef: + name: devlake-webhook-secret + key: token + source: | + apk add --no-cache jq curl + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + . as $root | + { + id: .after, + createdDate: (.commits | first | .timestamp), + startedDate: (.commits | first | .timestamp), + finishedDate: (.commits | last | .timestamp), + environment: "PRODUCTION", + result: "SUCCESS", + name: ("deployment-" + (.after | .[0:7])), + deploymentCommits: [ + .commits[] | { + repoUrl: $root.repository.clone_url, + refName: $root.ref, + startedDate: .timestamp, + finishedDate: .timestamp, + commitSha: .id, + commitMsg: .message, + result: "SUCCESS", + name: ("deployment-" + (.id | .[0:7])) + } + ] + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: pull-requests-workflow-template +spec: + entrypoint: process-pull-request + serviceAccountName: devlake-webhook-sa + arguments: + parameters: + - name: body + - name: headers + - name: url + templates: + - name: process-pull-request + inputs: + parameters: + - name: body + - name: headers + - name: url + script: + image: alpine + command: [sh] + env: + - name: DEVLAKE_TOKEN + valueFrom: + secretKeyRef: + name: devlake-webhook-secret + key: token + + source: | + apk add --no-cache jq curl + echo '{{inputs.parameters.body}}' + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + { + id: (.pull_request.id | tostring), + title: .pull_request.title, + description: .pull_request.body, + status: (if .pull_request.state == "open" then "OPEN" elif .pull_request.merged then "MERGED" else "CLOSED" end), + createdDate: .pull_request.created_at, + updatedDate: .pull_request.updated_at, + closedDate: .pull_request.closed_at, + mergedDate: .pull_request.merged_at, + type: "PR", + url: .pull_request.html_url, + authorId: .pull_request.user.login, + authorName: (.pull_request.user.name // .pull_request.user.login), + reviewers: [.pull_request.requested_reviewers[].login], + comments: (.pull_request.comments // 0), + commits: (.pull_request.commits // 0), + additions: (.pull_request.additions // 0), + deletions: (.pull_request.deletions // 0), + changedFiles: (.pull_request.changed_files // 0), + baseBranch: .pull_request.base.ref, + headBranch: .pull_request.head.ref, + repoId: .repository.id, + repoName: .repository.full_name + }') + echo "====Transformed Payload====" + echo "$PAYLOAD" | jq '.' + + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml deleted file mode 100644 index c8f5e7c80..000000000 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/gitea-external-secret.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: gitea-credentials - namespace: ${{values.namespace}} -spec: - secretStoreRef: - name: gitea - kind: ClusterSecretStore - refreshInterval: "0" - target: - name: gitea-credentials - data: - - secretKey: GITEA_USERNAME - remoteRef: - key: gitea-credential - property: username - - secretKey: GITEA_PASSWORD - remoteRef: - key: gitea-credential - property: password - - secretKey: GITEA_TOKEN - remoteRef: - key: gitea-credential - property: token \ No newline at end of file diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rbac.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rbac.yaml new file mode 100644 index 000000000..6c12dba4f --- /dev/null +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rbac.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: devlake-webhook-sa + namespace: ${{values.namespace}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: devlake-webhook-binding + namespace: ${{values.namespace}} +subjects: +- kind: ServiceAccount + name: devlake-webhook-sa + namespace: ${{values.namespace}} +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml deleted file mode 100644 index d1ed3409f..000000000 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rollout-notifs.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: argo-rollouts-notification-configmap - namespace: ${{values.namespace}} -data: - trigger.on-rollout-aborted: | - - send: [modengg-rollout-aborted] - trigger.on-rollout-completed: | - - send: [modengg-rollout-completed] - service.webhook.deployments: | - url: https://${{values.hostname}}/argo-events/deployments - template.modengg-rollout-aborted: | - - stuff - template.modengg-rollout-completed: | - - stuff - - diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml index 9d18bc9dd..4a1337c67 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml @@ -20,14 +20,15 @@ spec: script: image: alpine command: [sh] - env: - - name: DEVLAKE_TOKEN - valueFrom: - secretKeyRef: - name: devlake-webhook-secret - key: token + envFrom: + - secretRef: + name: devlake-webhook-secret + - configMapRef: + name: devlake-webhook-id source: | apk add --no-cache jq curl + webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/$DEVLAKE_HOOK_ID/deployments + echo '{{inputs.parameters.body}}' PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' { @@ -75,12 +76,11 @@ spec: script: image: alpine command: [sh] - env: - - name: DEVLAKE_TOKEN - valueFrom: - secretKeyRef: - name: devlake-webhook-secret - key: token + envFrom: + - secretRef: + name: devlake-webhook-secret + - configMapRef: + name: devlake-webhook-id source: | apk add --no-cache jq curl PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' @@ -133,13 +133,11 @@ spec: script: image: alpine command: [sh] - env: - - name: DEVLAKE_TOKEN - valueFrom: - secretKeyRef: - name: devlake-webhook-secret - key: token - + envFrom: + - secretRef: + name: devlake-webhook-secret + - configMapRef: + name: devlake-webhook-id source: | apk add --no-cache jq curl echo '{{inputs.parameters.body}}' From 83016cf870745ae5fd66872d2a2159d6154a13ee Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 2 Apr 2025 17:27:11 -0700 Subject: [PATCH 21/63] start move to templates for cicd --- .../cicd-pipeline/provisioner/wf-run.yaml | 4 +- .../skeleton/manifests/cicd-pipeline.yaml | 191 ++---------------- .../manifests}/wf-templates.yaml | 25 ++- .../cicd-pipeline/template-cicd-pipeline.yaml | 25 +-- 4 files changed, 46 insertions(+), 199 deletions(-) rename platform/backstage/templates/cicd-pipeline/{provisioner => skeleton/manifests}/wf-templates.yaml (94%) diff --git a/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml b/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml index 6725c6437..df566882f 100644 --- a/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml +++ b/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml @@ -55,8 +55,8 @@ spec: - - name: run-warmup templateRef: - name: cicd-cache-warmup-template - template: cicd-warmup + name: run-cicd-template + template: run-cicd arguments: parameters: - name: appname diff --git a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml index 6eb036f26..89ed4b6d9 100644 --- a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml @@ -70,180 +70,27 @@ spec: resources: requests: storage: 256Mi - #volumes: - #- name: cache - # persistentVolumeClaim: - # claimName: ${{values.appname}}-cache-vol templates: - - name: ci-build - dag: - tasks: - - name: clean - template: cleandir - - - name: clone - template: clonerepo - arguments: - parameters: - - name: apprepo - value: "{{workflow.parameters.apprepo}}" - - name: branch - value: "{{workflow.parameters.branch}}" - depends: "clean" - - - name: build-and-push-image - template: build-image-kaniko - arguments: - parameters: - - name: tag-version - value: "{{workflow.parameters.tag-version}}" - - name: dockerfile-path - value: "{{workflow.parameters.dockerfile-path}}" - depends: "clone" - - - name: update-and-push-image-version - template: update-image-tag - arguments: - parameters: - - name: env - value: dev - - name: tag - value: "{{workflow.parameters.tag-version}}" - - name: apprepo - value: "{{workflow.parameters.apprepo}}" - - name: deployment-path - value: "{{workflow.parameters.deployment-path}}" - depends: "build-and-push-image" - - - name: cleandir - container: - volumeMounts: - - mountPath: /workdir - name: workdir - image: alpine:3.20 - workingDir: /workdir - args: - - /bin/sh - - -c - - | - set -ex - if [ "$(ls -A /workdir)" ]; then - echo "Directory /workdir is not empty. Removing contents..." - rm -rf /workdir/* - fi - - - name: clonerepo - inputs: + - name: run-cicd + templateRef: + name: run-cicd-template + template: run-cicd + arguments: parameters: - - name: apprepo - - name: branch - container: - volumeMounts: - - mountPath: /workdir - name: workdir - image: alpine/git:v2.45.1 - workingDir: /workdir - args: - - clone - - --depth - - "1" - - --branch - - "{{inputs.parameters.branch}}" - - --single-branch - - "https://{{inputs.parameters.apprepo}}" - - -c - - http.sslVerify=false - - . - - - - name: build-image-kaniko - inputs: - parameters: - - name: tag-version - - name: dockerfile-path - volumes: - - name: docker-config - secret: - secretName: "{{workflow.parameters.aws_region}}-regcred" - container: - image: gcr.io/kaniko-project/executor:debug - volumeMounts: - - name: workdir - mountPath: /workdir - - name: docker-config - mountPath: /kaniko/.docker - # - name: cache - # mountPath: /cache - workingDir: /workdir - resources: - limits: - cpu: 2 - memory: 5Gi - env: - - name: DOCKER_CONFIG - value: /kaniko/.docker - - name: ECR_REPO - valueFrom: - configMapKeyRef: - name: "{{workflow.parameters.appname}}-ecr-repo" - key: image - command: [ - "/busybox/sh", - "-c", - "/kaniko/executor \ - --dockerfile=Dockerfile \ - --context={{inputs.parameters.dockerfile-path}} \ - --destination=${ECR_REPO}:{{inputs.parameters.tag-version}} \ - --cache=true \ - --skip-tls-verify" - ] - - - name: update-image-tag - inputs: - parameters: - - name: env - - name: tag - - name: apprepo - - name: deployment-path - container: - envFrom: - - secretRef: - name: gitea-credentials - - configMapRef: - name: "{{workflow.parameters.appname}}-ecr-repo" - volumeMounts: - - mountPath: /workdir - name: workdir - image: alpine/git:v2.45.2 - workingDir: /workdir - command: ['/bin/sh'] - args: - - -c - - | - apk add --no-cache sed - set -ex - DEPLOY_PATH="{{inputs.parameters.deployment-path}}/{{inputs.parameters.env}}" - tag="{{inputs.parameters.tag}}" - mkdir -p $DEPLOY_PATH - git config --global user.email "noreply@aws.com" - git config --global user.name ARGO - git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git - git pull --rebase - echo "$image:{{inputs.parameters.tag}}" > ./deployment/"{{inputs.parameters.env}}"/image.txt - if [ -f $DEPLOY_PATH/application.yml ]; then - sed -i 's+image: .*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" - sed -i 's+image: '"$image"'.*+image: '"${image}:${tag}"'+g' "${DEPLOY_PATH}/application.yml" - fi - m_count=$(git status -s | wc -l) - if [ $m_count -ge 1 ]; then - git add "{{inputs.parameters.deployment-path}}" - image_version="{{inputs.parameters.tag}}" - git commit -m "[SKIP CI] Updated image version to $image_version" - git push --set-upstream origin main - else - echo "No updates to values file, skipping git push" - fi - + - name: appname + value: "{{workflow.parameters.appname}}" + - name: apprepo + value: "{{workflow.parameters.apprepo}}" + - name: tag-version + value: "{{workflow.parameters.tag-version" + - name: branch + value: "{{workflow.parameters.branch}}" + - name: aws_region + value: "{{workflow.parameters.aws_region}}" + - name: dockerfile_path + value: "{{workflow.parameters.dockerfile_path}}" + - name: deployment_path + value: "{{workflow.parameters.deployment_path}}" parameters: - src: dependencyName: source-dep diff --git a/platform/backstage/templates/cicd-pipeline/provisioner/wf-templates.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml similarity index 94% rename from platform/backstage/templates/cicd-pipeline/provisioner/wf-templates.yaml rename to platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml index e62008169..17c1fab12 100644 --- a/platform/backstage/templates/cicd-pipeline/provisioner/wf-templates.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml @@ -122,7 +122,7 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: cicd-cache-warmup-template + name: run-cicd-template namespace: ${{values.namespace}} spec: arguments: @@ -131,6 +131,10 @@ spec: value: ${{values.appname}} - name: apprepo value: ${{values.hostname}}/gitea/giteaAdmin/${{values.appname}}-cicd + - name: branch + value: main + - name: tag-version + value: init - name: aws_region value: ${{values.aws_region}} - name: dockerfile_path @@ -139,7 +143,7 @@ spec: value: ${{values.deployment_path}} templates: - - name: cicd-warmup + - name: run-cicd dag: tasks: - name: clean @@ -152,17 +156,17 @@ spec: - name: apprepo value: "{{workflow.parameters.apprepo}}" - name: branch - value: main + value: "{{workflow.parameters.branch}}" depends: "clean" - - name: warmup - template: build-image-for-cache + - name: build-image + template: build-image arguments: parameters: - name: dockerfile-path value: "{{workflow.parameters.dockerfile_path}}" - name: tag-version - value: init + value: "{{workflow.parameters.tag-version}}" depends: "clone" - name: edit-tag @@ -172,9 +176,11 @@ spec: - name: env value: dev - name: tag - value: init + value: "{{workflow.parameters.tag-version}}" - name: apprepo value: "{{workflow.parameters.apprepo}}" + - name: branch + value: "{{workflow.parameters.branch}}" - name: deployment-path value: "{{workflow.parameters.deployment_path}}" depends: "warmup" @@ -219,7 +225,7 @@ spec: - http.sslVerify=false - . - - name: build-image-for-cache + - name: build-image inputs: parameters: - name: tag-version @@ -265,6 +271,7 @@ spec: - name: env - name: tag - name: apprepo + - name: branch - name: deployment-path container: envFrom: @@ -300,7 +307,7 @@ spec: git add "{{inputs.parameters.deployment-path}}" image_version="{{inputs.parameters.tag}}" git commit -m "[SKIP CI] Updated image version to $image_version" - git push --set-upstream origin main + git push --set-upstream origin "{{inputs.parameters.branch}}" else echo "No updates to values file, skipping git push" fi diff --git a/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml b/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml index f5e7104b5..105f6c272 100644 --- a/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml +++ b/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml @@ -80,22 +80,6 @@ spec: input: seconds: 15 - - action: cnoe:kubernetes:apply - name: Apply Workflow Templates - id: apply-temp - input: - namespaced: true - manifestPath: wf-templates.yaml - clusterName: local - - - action: cnoe:kubernetes:apply - name: Run Workflow - id: run-wf - input: - namespaced: true - manifestPath: wf-run.yaml - clusterName: local - - id: create-argocd-app name: Create ArgoCD App action: cnoe:create-argocd-app @@ -107,6 +91,15 @@ spec: # necessary until we generate our own cert repoUrl: http://my-gitea-http.gitea.svc.cluster.local:3000/giteaAdmin/${{parameters.appname}}-cicd path: "manifests" + + - action: cnoe:kubernetes:apply + name: Run Workflow + id: run-wf + input: + namespaced: true + manifestPath: wf-run.yaml + clusterName: local + - id: register name: Register action: catalog:register From 76c3a7434ccdcbf883160e664883b3340f6a64ab Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 7 Apr 2025 13:32:53 -0700 Subject: [PATCH 22/63] changes for move to rds --- .../components/rds-cluster-mysql.yaml | 69 +++++++++ packages/devlake/base/values.yaml | 30 ++-- packages/devlake/dev/kustomization.yaml | 5 + packages/devlake/dev/mysql-init-job.yaml | 69 ++++++++- .../devlake/dev/patches/devlake-patch-sa.yaml | 9 ++ packages/devlake/dev/rds-mysql.yaml | 41 +++++ .../skeleton/manifests/cicd-pipeline.yaml | 55 ++++--- .../provisioner/wf-templates.yaml | 24 ++- .../skeleton/manifests/events.yaml | 8 + .../skeleton/manifests/exit-handler.yaml | 146 +++++++++--------- .../skeleton/manifests/wf-templates.yaml | 72 +-------- platform/components/rds-cluster-mysql.cue | 73 +++++++++ platform/infra/terraform/bootstrap/main.tf | 13 +- .../infra/terraform/bootstrap/variables.tf | 5 +- .../mgmt/terraform/mgmt-cluster/outputs.tf | 5 + .../mgmt/terraform/mgmt-cluster/vpc.tf | 7 +- .../infra/terraform/setup-environments.sh | 5 +- 17 files changed, 445 insertions(+), 191 deletions(-) create mode 100644 deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml create mode 100644 packages/devlake/dev/patches/devlake-patch-sa.yaml create mode 100644 packages/devlake/dev/rds-mysql.yaml create mode 100644 platform/components/rds-cluster-mysql.cue diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml new file mode 100644 index 000000000..5674bfb3d --- /dev/null +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -0,0 +1,69 @@ +apiVersion: core.oam.dev/v1beta1 +kind: ComponentDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Amazon RDS MySQL Cluster + name: rds-cluster-mysql +spec: + schematic: + cue: + template: | + output: { + apiVersion: "rds.aws.upbound.io/v1beta1" + kind: "Cluster" + metadata: name: "\(context.name)-cluster" + spec: { + forProvider: { + autoGeneratePassword: true + engine: "aurora-mysql" + masterPasswordSecretRef: { + key: "password" + name: "\(context.name)-cluster-password" + namespace: "vela-system" + } + masterUsername: "awsrdsadmin" + region: "\(parameter.region)" + dbSubnetGroupNameRef: name: "\(context.name)-subnetgroup" + skipFinalSnapshot: true + } + writeConnectionSecretToRef: { + name: "\(context.name)-cluster-connection" + namespace: "vela-system" + } + } + } + outputs: { + "\(context.name)-subnetgroup": { + apiVersion: "rds.aws.upbound.io/v1beta1" + kind: "SubnetGroup" + metadata: name: "\(context.name)-subnetgroup" + spec: forProvider: { + region: "\(parameter.region)" + subnetIds: [ for subnetId in parameter.subnetIds {"\(subnetId)"}] + } + } + + "\(context.name)-instance": { + apiVersion: "rds.aws.upbound.io/v1beta1" + kind: "ClusterInstance" + metadata: name: "\(context.name)-clusterinstance" + spec: forProvider: { + region: "\(parameter.region)" + clusterIdentifierRef: name: "\(context.name)-cluster" + engine: "aurora-mysql" + instanceClass: "db.r5.large" + } + } + + } + + parameter: { + region: string + subnetIds: [...string] + } + workload: + definition: + apiVersion: apps/v1 + kind: Deployment + diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 5a03514f7..3aacc9061 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -2,29 +2,17 @@ grafana: enabled: false external: - url: "http://localhost:4000" # Set to AMG Endpoint in setup-environments + url: "http://localhost:4000" # Set to AMG Endpoint in setup-environments - doesn't work - need some sort of ingress maybe? mysql: - storage: - class: gp3 - - service: - type: "NodePort" - nodePort: "30306" - - securityContext: - runAsNonRoot: true - runAsUser: 999 - runAsGroup: 999 - fsGroup: 999 - - containerSecurityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault + useExternal: true + externalServer: "${EXTERNAL_SERVER}" + externalPort: "${EXTERNAL_PORT}" + +option: + database: mysql + connectionSecretName: "devlake-mysql-auth" + autoCreateSecret: false lake: diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 4e374eab6..1a4834c3a 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -1,3 +1,8 @@ resources: - external-secrets.yaml - mysql-init-job.yaml + - rds-mysql.yaml + +patchesStrategicMerge: + - patches/devlake-patch-sa.yaml + diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml index c509f3d21..6f13bd6cd 100644 --- a/packages/devlake/dev/mysql-init-job.yaml +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -1,3 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: devlake-mysql-access-binding + namespace: devlake +subjects: +- kind: ServiceAccount + name: devlake-mysql-access + namespace: devlake +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- apiVersion: batch/v1 kind: Job metadata: @@ -7,7 +21,28 @@ spec: metadata: name: mysql-user-setup spec: + serviceAccountName: devlake-mysql-access restartPolicy: OnFailure + initContainers: + - name: create-secret + image: bitnami/kubectl + command: + - /bin/bash + - -c + - | + kubectl create secret generic devlake-mysql-auth \ + --from-literal=MYSQL_USER=merico \ + --from-literal=MYSQL_PASSWORD=merico \ + --from-literal=MYSQL_DATABASE=lake \ + --from-literal=MYSQL_ROOT_PASSWORD=$password \ + --from-literal=DB_URL=$endpoint + envFrom: + - secretRef: + name: devlake-mysql-db-cluster-password + namespace: vela-system + - secretRef: + name: devlake-mysql-db-cluster-connection + namespace: vela-system containers: - name: create-grafana-user image: mysql:8 @@ -15,13 +50,24 @@ spec: - /bin/bash - -c - | - until mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do + until mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do echo "Waiting for MySQL to be ready..." sleep 2 done + + mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e " + CREATE DATABASE IF NOT EXISTS lake; + " + + echo "Creating devlake user..." + mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e " + CREATE USER IF NOT EXISTS 'merico' IDENTIFIED BY 'merico'; + GRANT ALL PRIVILEGES ON lake.* TO 'merico'; + FLUSH PRIVILEGES; + " echo "Creating Grafana user..." - mysql -h devlake-mysql -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e " CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password'; GRANT SELECT ON lake.* TO 'grafanaReader'; FLUSH PRIVILEGES; @@ -31,4 +77,23 @@ spec: envFrom: - secretRef: name: devlake-mysql-auth + - name: create-secret + image: bitnami/kubectl + command: + - /bin/bash + - -c + - | + kubectl create secret generic devlake-mysql-auth \ + --from-literal=MYSQL_USER=merico \ + --from-literal=MYSQL_PASSWORD=merico \ + --from-literal=MYSQL_DATABASE=lake \ + --from-literal=MYSQL_ROOT_PASSWORD=$password \ + --from-literal=DB_URL=$url + envFrom: + - secretRef: + name: devlake-mysql-db-cluster-password + namespace: vela-system + - secretRef: + name: devlake-mysql-db-cluster-connection + namespace: vela-system diff --git a/packages/devlake/dev/patches/devlake-patch-sa.yaml b/packages/devlake/dev/patches/devlake-patch-sa.yaml new file mode 100644 index 000000000..0a6a3f300 --- /dev/null +++ b/packages/devlake/dev/patches/devlake-patch-sa.yaml @@ -0,0 +1,9 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: devlake-lake + namespace: devlake +spec: + template: + spec: + serviceAccountName: devlake-mysql-access diff --git a/packages/devlake/dev/rds-mysql.yaml b/packages/devlake/dev/rds-mysql.yaml new file mode 100644 index 000000000..16c3cd6d1 --- /dev/null +++ b/packages/devlake/dev/rds-mysql.yaml @@ -0,0 +1,41 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: devlake-rds-mysql + namespace: devlake +spec: + components: + - name: devlake-mysql-db + type: rds-cluster-mysql + properties: + region: us-west-2 + subnetIds: + - subnet-xxxxxx1 + - subnet-xxxxxx2 + - subnet-xxxxxx3 + traits: + - type: component-iam-policy + properties: + service: rds-db + policy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "rds-db:connect" + ], + "Resource": "*" + } + ] + } + + + - name: devlake-mysql-access + type: dp-service-account + properties: + clusterName: modern-engineering + clusterRegion: us-west-2 + componentNamesForAccess: + - devlake-mysql-db diff --git a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml index 89ed4b6d9..534536780 100644 --- a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml @@ -44,7 +44,7 @@ spec: entity-id: ${{values.appname}} spec: serviceAccountName: ${{values.appname}}-cicd-sa - entrypoint: ci-build + entrypoint: run-cicd arguments: parameters: - name: apprepo @@ -61,6 +61,10 @@ spec: value: ${{values.deployment_path}} - name: appname value: ${{values.appname}} + - name: body + value: "" + - name: headers + value: "" volumeClaimTemplates: - metadata: name: workdir @@ -71,31 +75,40 @@ spec: requests: storage: 256Mi templates: - - name: run-cicd - templateRef: - name: run-cicd-template - template: run-cicd - arguments: - parameters: - - name: appname - value: "{{workflow.parameters.appname}}" - - name: apprepo - value: "{{workflow.parameters.apprepo}}" - - name: tag-version - value: "{{workflow.parameters.tag-version" - - name: branch - value: "{{workflow.parameters.branch}}" - - name: aws_region - value: "{{workflow.parameters.aws_region}}" - - name: dockerfile_path - value: "{{workflow.parameters.dockerfile_path}}" - - name: deployment_path - value: "{{workflow.parameters.deployment_path}}" + steps: + - - name: run-cicd + templateRef: + name: run-cicd-template + template: run-cicd + arguments: + parameters: + - name: appname + value: "{{workflow.parameters.appname}}" + - name: apprepo + value: "{{workflow.parameters.apprepo}}" + - name: tag-version + value: "{{workflow.parameters.tag-version" + - name: branch + value: "{{workflow.parameters.branch}}" + - name: aws_region + value: "{{workflow.parameters.aws_region}}" + - name: dockerfile_path + value: "{{workflow.parameters.dockerfile_path}}" + - name: deployment_path + value: "{{workflow.parameters.deployment_path}}" parameters: - src: dependencyName: source-dep dataTemplate: "{{ .Input.body.head_commit.id }}" dest: spec.arguments.parameters.2.value + - src: + dependencyName: source-dep + dataTemplate: "{{ .Input.body }}" + dest: spec.arguments.parameters.7.value + - src: + dependencyName: source-dep + dataTemplate: "{{ .Input.headers }}" + dest: spec.arguments.parameters.8.value --- apiVersion: v1 kind: Service diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml index 1b09e9798..5232e6057 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml @@ -203,13 +203,31 @@ spec: git config --global user.name ARGO git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git git pull --rebase - if [ -f manifests/cicd-pipeline.yml ]; then - yq 'select(.kind == "Sensor").spec.triggers[].template.argoWorkflow.source.resource.spec.onExit = "{{inputs.parameters.exit-hook-name}}"' -i manifests/cicd-pipeline.yaml + if [ -f manifests/cicd-pipeline.yaml ]; then + yq -i '.spec.triggers[0].template.argoWorkflow.source.resource.spec.templates[0].steps += [[{ + "name": "send-deployment-status", + "templateRef": { + "name": "cicd-deployments-dora-processing-template", + "template": "handle-rollout-check-and-process-deploy" + }, + "arguments": { + "parameters": [ + { + "name": "body", + "value": "{{workflow.parameters.body}}" + }, + { + "name": "headers", + "value": "{{workflow.parameters.headers}}" + } + ] + } + }]]' manifests/cicd-pipeline.yaml fi m_count=$(git status -s | wc -l) if [ $m_count -ge 1 ]; then git add . - git commit -m "Add Exit Handler for DORA Metrics" + git commit -m "Add Deployment Handler for DORA Metrics" git push --set-upstream origin main else echo "No updates, skipping git push" diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml index 4a19c1128..a7c03c750 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml @@ -37,6 +37,7 @@ spec: pull-requests-webhook: port: "12000" endpoint: "/pull-requests" + method: POST --- apiVersion: argoproj.io/v1alpha1 @@ -69,6 +70,8 @@ spec: metadata: generateName: incidents-workflow- spec: + serviceAccountName: devlake-webhook-sa + entrypoint: process-incident workflowTemplateRef: name: incidents-workflow-template arguments: @@ -112,6 +115,10 @@ spec: dependencyName: deployments-webhook dataKey: body dest: spec.arguments.parameters.0.value + - src: + dependencyName: deployments-webhook + dataKey: headers + dest: spec.arguments.parameters.1.value - template: name: pull-requests-workflow-trigger @@ -127,6 +134,7 @@ spec: spec: workflowTemplateRef: name: pull-requests-workflow-template + entrypoint: process-pull-request arguments: parameters: - name: body diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml index c7632e84b..a1a5c572f 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml @@ -1,34 +1,65 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: cicd-deployments-exit-handler + name: cicd-deployments-dora-processing-template + namespace: ${{values.namespace}} spec: - entrypoint: handle-exit + entrypoint: handle-rollout-check-and-process-deploy serviceAccountName: devlake-webhook-sa + volumes: + - name: argocd-cluster-secret + secret: + secretName: dev-cluster-argo-secret + namespace: argocd arguments: parameters: - name: body - name: headers - name: url + value: devlake-ui.devlake.svc.cluster.local:4000 + - name: appname + value: ${{values.appName}} + - name: namespace + value: ${{values.appNamespace}} templates: + - name: handle-rollout-check-and-process-deploy + steps: + - - name: check-rollout + template: check-rollout-status + - - name: process-and-send-deployment + template: process-deployment + arguments: + parameters: + - name: status + value: "{{steps.check-rollout.outputs.result}}" + - name: body + value: "{{workflow.parameters.body}}" + - name: url + value: "{{workflow.parameters.url}}" + - name: headers + value: "{{workflow.parameters.headers}}" + + - name: process-deployment inputs: parameters: - name: body - name: headers - name: url + - name: status script: image: alpine command: [sh] - env: - - name: DEVLAKE_TOKEN - valueFrom: - secretKeyRef: - name: devlake-webhook-secret - key: token + envFrom: + - secretRef: + name: devlake-webhook-secret + - configMapRef: + name: devlake-webhook-id source: | apk add --no-cache jq curl - PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' + RESULT="{{inputs.parameters.status}} + webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/${DEVLAKE_HOOK_ID}/deployments + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq --arg result "$RESULT" ' . as $root | { id: .after, @@ -36,7 +67,7 @@ spec: startedDate: (.commits | first | .timestamp), finishedDate: (.commits | last | .timestamp), environment: "PRODUCTION", - result: "SUCCESS", + result: $result, name: ("deployment-" + (.after | .[0:7])), deploymentCommits: [ .commits[] | { @@ -54,67 +85,42 @@ spec: echo "====Transformed Payload====" echo "$PAYLOAD" | jq '.' - curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: pull-requests-workflow-template -spec: - entrypoint: process-pull-request - serviceAccountName: devlake-webhook-sa - arguments: - parameters: - - name: body - - name: headers - - name: url - templates: - - name: process-pull-request - inputs: - parameters: - - name: body - - name: headers - - name: url + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" $webhook_url + + - name: check-rollout-status script: - image: alpine - command: [sh] - env: - - name: DEVLAKE_TOKEN - valueFrom: - secretKeyRef: - name: devlake-webhook-secret - key: token - + image: bitnami/kubectl:latest + command: [bash] + volumeMounts: + - name: argocd-cluster-secret + mountPath: /tmp/argocd-cluster + readOnly: true source: | - apk add --no-cache jq curl - echo '{{inputs.parameters.body}}' - PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' - { - id: (.pull_request.id | tostring), - title: .pull_request.title, - description: .pull_request.body, - status: (if .pull_request.state == "open" then "OPEN" elif .pull_request.merged then "MERGED" else "CLOSED" end), - createdDate: .pull_request.created_at, - updatedDate: .pull_request.updated_at, - closedDate: .pull_request.closed_at, - mergedDate: .pull_request.merged_at, - type: "PR", - url: .pull_request.html_url, - authorId: .pull_request.user.login, - authorName: (.pull_request.user.name // .pull_request.user.login), - reviewers: [.pull_request.requested_reviewers[].login], - comments: (.pull_request.comments // 0), - commits: (.pull_request.commits // 0), - additions: (.pull_request.additions // 0), - deletions: (.pull_request.deletions // 0), - changedFiles: (.pull_request.changed_files // 0), - baseBranch: .pull_request.base.ref, - headBranch: .pull_request.head.ref, - repoId: .repository.id, - repoName: .repository.full_name - }') - echo "====Transformed Payload====" - echo "$PAYLOAD" | jq '.' + mkdir -p ~/.kube + cp /tmp/argocd-cluster/config ~/.kube/config + + # Install kubectl-argo-rollouts plugin + curl -LO https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64 + chmod +x ./kubectl-argo-rollouts-linux-amd64 + mv ./kubectl-argo-rollouts-linux-amd64 /usr/local/bin/kubectl-argo-rollouts + + SEARCH_WORD="{{workflow.parameters.appname}}" + ROLLOUT_NAME=$(kubectl get rollout -n {{workflow.parameters.namespace}} -o json | \ + jq -r --arg WORD "$SEARCH_WORD" '.items[] | select(.metadata.name | contains($WORD)) | .metadata.name') - curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + if [ -z "$ROLLOUT_NAME" ]; then + echo "No rollout found containing word: $SEARCH_WORD" + exit 1 + fi + # Initialize timeout variables + TIMEOUT=600 # 10 minutes timeout + INTERVAL=10 # Check every 10 seconds + ELAPSED=0 + if kubectl argo rollouts status $ROLLOUT_NAME -n "{{workflow.parameters.namespace}}" --timeout 2m; then + echo "SUCCESS" + exit 0 + else + echo "FAILURE" + exit 0 + fi diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml index 4a1337c67..7e31111b3 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml +++ b/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml @@ -2,9 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: incidents-workflow-template + namespace: ${{values.namespace}} spec: - entrypoint: process-incident - serviceAccountName: devlake-webhook-sa arguments: parameters: - name: body @@ -27,7 +26,7 @@ spec: name: devlake-webhook-id source: | apk add --no-cache jq curl - webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/$DEVLAKE_HOOK_ID/deployments + webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/${DEVLAKE_HOOK_ID}/issues echo '{{inputs.parameters.body}}' PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' @@ -52,72 +51,14 @@ spec: echo "====Transformed Payload====" echo "$PAYLOAD" | jq '.' - curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: deployments-workflow-template -spec: - entrypoint: process-deployment - serviceAccountName: devlake-webhook-sa - arguments: - parameters: - - name: body - - name: headers - - name: url - templates: - - name: process-deployment - inputs: - parameters: - - name: body - - name: headers - - name: url - script: - image: alpine - command: [sh] - envFrom: - - secretRef: - name: devlake-webhook-secret - - configMapRef: - name: devlake-webhook-id - source: | - apk add --no-cache jq curl - PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' - . as $root | - { - id: .after, - createdDate: (.commits | first | .timestamp), - startedDate: (.commits | first | .timestamp), - finishedDate: (.commits | last | .timestamp), - environment: "PRODUCTION", - result: "SUCCESS", - name: ("deployment-" + (.after | .[0:7])), - deploymentCommits: [ - .commits[] | { - repoUrl: $root.repository.clone_url, - refName: $root.ref, - startedDate: .timestamp, - finishedDate: .timestamp, - commitSha: .id, - commitMsg: .message, - result: "SUCCESS", - name: ("deployment-" + (.id | .[0:7])) - } - ] - }') - echo "====Transformed Payload====" - echo "$PAYLOAD" | jq '.' - - curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" $webhook_url --- apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: pull-requests-workflow-template + namespace: ${{values.namespace}} spec: - entrypoint: process-pull-request - serviceAccountName: devlake-webhook-sa arguments: parameters: - name: body @@ -140,7 +81,8 @@ spec: name: devlake-webhook-id source: | apk add --no-cache jq curl - echo '{{inputs.parameters.body}}' + webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/${DEVLAKE_HOOK_ID}/pull_requests + PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' { id: (.pull_request.id | tostring), @@ -169,5 +111,5 @@ spec: echo "====Transformed Payload====" echo "$PAYLOAD" | jq '.' - curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" '{{inputs.parameters.url}}' + curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" $webhook_url diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue new file mode 100644 index 000000000..c89376eef --- /dev/null +++ b/platform/components/rds-cluster-mysql.cue @@ -0,0 +1,73 @@ +"rds-cluster-mysql": { + alias: "" + annotations: {} + attributes: workload: definition: { + apiVersion: "apps/v1" + kind: "Deployment" + } + description: "Amazon RDS MySQL Cluster" + labels: {} + type: "component" +} + +template: { + output: { + apiVersion: "rds.aws.upbound.io/v1beta1" + kind: "Cluster" + metadata: { + name: "\(context.name)-cluster" + } + spec: { + forProvider: { + autoGeneratePassword: true + engine: "aurora-mysql" + masterPasswordSecretRef: { + key: "password" + name: "\(context.name)-cluster-password" + namespace: "vela-system" + } + masterUsername: "awsrdsadmin" + region: "\(parameter.region)" + dbSubnetGroupNameRef: name: "\(context.name)-subnetgroup" + skipFinalSnapshot: true + } + writeConnectionSecretToRef: { + name: "\(context.name)-cluster-connection" + namespace: "vela-system" + } + } + } + outputs: { + "\(context.name)-subnetgroup": { + apiVersion: "rds.aws.upbound.io/v1beta1" + kind: "SubnetGroup" + metadata: name: "\(context.name)-subnetgroup" + spec: { + forProvider: { + region: "\(parameter.region)" + subnetIds: [ for subnetId in parameter.subnetIds {"\(subnetId)"}] + } + } + } + + "\(context.name)-instance": { + apiVersion: "rds.aws.upbound.io/v1beta1" + kind: "ClusterInstance" + metadata: name: "\(context.name)-clusterinstance" + spec: { + forProvider: { + region: "\(parameter.region)" + clusterIdentifierRef: name: "\(context.name)-cluster" + engine: "aurora-mysql" + instanceClass: "db.r5.large" + } + } + } + + } + + parameter: { + region: string + subnetIds: [...string] + } +} diff --git a/platform/infra/terraform/bootstrap/main.tf b/platform/infra/terraform/bootstrap/main.tf index 639f7fd04..7b738c706 100644 --- a/platform/infra/terraform/bootstrap/main.tf +++ b/platform/infra/terraform/bootstrap/main.tf @@ -99,10 +99,19 @@ module "managed_grafana" { # Dummy values for SAML configuration to setup will be updated after keycloak integration saml_idp_metadata_url = var.grafana_keycloak_idp_url - create_security_group = false + create_security_group = true + security_group_rules = { + egress_mysql = { + description = "Allow egress to MySQL" + from_port = 3306 + to_port = 3306 + protocol = "tcp" + cidr_blocks = var.vpc_cidr + } + } + vpc_configuration = { subnet_ids = var.eks_cluster_private_subnets - security_group_ids = [var.eks_cluster_node_security_group_id] } tags = local.tags diff --git a/platform/infra/terraform/bootstrap/variables.tf b/platform/infra/terraform/bootstrap/variables.tf index 385c83774..a69bb960f 100644 --- a/platform/infra/terraform/bootstrap/variables.tf +++ b/platform/infra/terraform/bootstrap/variables.tf @@ -21,8 +21,7 @@ variable "eks_cluster_private_subnets" { type = list(string) } -variable "eks_cluster_node_security_group_id" { - description = "VPC security groups for AMG configuration" +variable "vpc_cidr" { + description = "VPC CIDR" type = string - } diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf index 7fd801dc8..7297f36d5 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf @@ -18,6 +18,11 @@ output "vpc_cidr" { value = local.vpc_cidr } +output "database_subnet_ids" { + description = "VPC Database Subnet Ids" + value = module.vpc.database_subnets +} + output "availability_zones" { description = "Default Availability Zone of the VPC created by the Module" value = local.azs diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf index 1362ee3d2..208e06d9c 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf @@ -7,11 +7,16 @@ module "vpc" { azs = local.azs private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 8)] public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] enable_nat_gateway = true single_nat_gateway = true + create_database_subnet_route_table = true + create_database_internet_gateway_route = false + create_database_nat_gateway_route = true + public_subnet_tags = { "kubernetes.io/role/elb" = 1 } @@ -23,4 +28,4 @@ module "vpc" { } tags = local.tags -} \ No newline at end of file +} diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index ace205c59..95fbe9b15 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -62,7 +62,6 @@ ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) -export TF_eks_cluster_node_security_group_id=$(terraform output -raw eks_cluster_node_security_group_id) echo "private subnets are : " $TF_eks_cluster_private_subnets # For Database and EC2 database export TF_eks_cluster_vpc_cidr=$(terraform output -raw vpc_cidr) @@ -122,8 +121,8 @@ git clone https://github.com/aws-observability/terraform-aws-observability-accel echo "bootstrapping Terraform" terraform -chdir=bootstrap init -reconfigure -terraform -chdir=bootstrap plan -var eks_cluster_private_subnets="$TF_eks_cluster_private_subnets" -var eks_cluster_node_security_group_id="$TF_eks_cluster_node_security_group_id" -terraform -chdir=bootstrap apply -var eks_cluster_private_subnets="$TF_eks_cluster_private_subnets" -var eks_cluster_node_security_group_id="$TF_eks_cluster_node_security_group_id" \ +terraform -chdir=bootstrap plan -var eks_cluster_private_subnets="$TF_eks_cluster_private_subnets" -var vpc_cidr="$TF_eks_cluster_vpc_cidr" +terraform -chdir=bootstrap apply -var eks_cluster_private_subnets="$TF_eks_cluster_private_subnets" -var vpc_cidr="$TF_eks_cluster_vpc_cidr" \ -auto-approve export TF_VAR_state_s3_bucket=$(terraform -chdir=bootstrap output -raw eks-accelerator-bootstrap-state-bucket) From feacb2e3c519588318315d388d6b0208c695f033 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 14:32:35 -0700 Subject: [PATCH 23/63] setup environment config for crossplane on mgmt cluster add gitignore fix ordering --- .../mgmt/terraform/scripts/crossplane/.gitignore | 1 + .../scripts/crossplane/cp-mgmt-env-config.yaml | 13 +++++++++++++ platform/infra/terraform/setup-environments.sh | 12 +++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore create mode 100644 platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore new file mode 100644 index 000000000..1cda54be9 --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore @@ -0,0 +1 @@ +*.yml diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml new file mode 100644 index 000000000..c44ed240c --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.crossplane.io/v1alpha1 +kind: EnvironmentConfig +metadata: + name: mgmt-environment +data: + locations: + us: us-west-2 + privateSubnetIds: + - $PRIV_SUBNET_ID_1 + - $PRIV_SUBNET_ID_2 + - $PRIV_SUBNET_ID_3 + vpcId: $VPC_ID + diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 2da83da98..2c07cfa71 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -55,11 +55,21 @@ sed -e "s/INGRESS_DNS/${DNS_HOSTNAME}/g" ${REPO_ROOT}/platform/infra/terraform/m export GITHUB_URL=$(yq '.repo_url' ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/config.yaml) export GITHUB_BRANCH=$(yq '.repo_branch' ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/config.yaml) - # Deploy the apps on IDP Builder and ArgoCD ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ + +VPC_ID=$(terraform output -raw eks_cluster_vpc_id) + +PRIV_SUBNET_ID_1=$(terraform output -json eks_cluster_private_subnets | jq -r '.[0]') +PRIV_SUBNET_ID_2=$(terraform output -json eks_cluster_private_subnets | jq -r '.[1]') +PRIV_SUBNET_ID_3=$(terraform output -json eks_cluster_private_subnets | jq -r '.[2]') + +# Setup CP Env Config for MGMT Cluster +sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml +kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml + export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) echo "private subnets are : " $TF_eks_cluster_private_subnets From 50a3e8260625003571ee1168b9a5bf0c2874386e Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Fri, 18 Apr 2025 12:21:39 -0700 Subject: [PATCH 24/63] add readme about env config --- .../terraform/mgmt/terraform/scripts/crossplane/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md new file mode 100644 index 000000000..c464c6151 --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md @@ -0,0 +1,7 @@ +# How Environment Config works + +After VPC is created, the values for VPC Id and Subnet Ids in the environment config are substituted in the `setup-environments.sh` script. + +This enables any Crossplane XRDs created in the crossplane compositions on the management cluster to substitute these values into managed resources. This ensures that customers will not need to manually go in and specify these values when creating a resource using the XRDs. + +There is an example in the `mysql-aurora` XRD merged in from the Measuring Platform Success feature branch. From 4691ca37851be52eb2ea732d8d333560daa96cff Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 7 Apr 2025 13:54:14 -0700 Subject: [PATCH 25/63] fix subnets and yq fix yq fix for testing fixes fix policy fix policy again fix perms fix jobs fix job fix job fix values fix external some fixes for backstage test fixes --- .../templates/traits/component-policy.yaml | 2 +- packages/devlake/base/values.yaml | 4 +- packages/devlake/dev/kustomization.yaml | 4 +- packages/devlake/dev/mysql-init-job.yaml | 37 ++++--------------- .../backstage/templates/catalog-info.yaml | 2 +- .../provisioner/generate-data-linux.sh | 0 .../provisioner/generate-data.sh | 0 .../provisioner/init.sh | 0 .../provisioner/rbac.yaml | 0 .../provisioner/wf-run.yaml | 0 .../provisioner/wf-templates.yaml | 2 +- .../skeleton/catalog-info.yaml | 0 .../skeleton/manifests/events.yaml | 0 .../skeleton/manifests/exit-handler.yaml | 0 .../skeleton/manifests/rbac.yaml | 0 .../skeleton/manifests/wf-templates.yaml | 0 platform/infra/terraform/bootstrap/main.tf | 2 +- .../mgmt/terraform/mgmt-cluster/vpc.tf | 4 +- platform/traits/component-policy.cue | 2 +- 19 files changed, 18 insertions(+), 41 deletions(-) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/provisioner/generate-data-linux.sh (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/provisioner/generate-data.sh (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/provisioner/init.sh (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/provisioner/rbac.yaml (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/provisioner/wf-run.yaml (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/provisioner/wf-templates.yaml (99%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/skeleton/catalog-info.yaml (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/skeleton/manifests/events.yaml (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/skeleton/manifests/exit-handler.yaml (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/skeleton/manifests/rbac.yaml (100%) rename platform/backstage/templates/devlake-onboarding/{templates-devlake-onboarding => }/skeleton/manifests/wf-templates.yaml (100%) diff --git a/deployment/addons/kubevela/templates/traits/component-policy.yaml b/deployment/addons/kubevela/templates/traits/component-policy.yaml index 2948b1488..8aef16608 100644 --- a/deployment/addons/kubevela/templates/traits/component-policy.yaml +++ b/deployment/addons/kubevela/templates/traits/component-policy.yaml @@ -48,7 +48,7 @@ spec: metadata: name: "\(context.appName)-\(context.name)-iam-policy" spec: { name: "\(context.appName)-\(context.name)-iam-policy" - forProvider: policy: "\(policy)" + forProvider: policy: "\(parameter.policy)" } } } diff --git a/packages/devlake/base/values.yaml b/packages/devlake/base/values.yaml index 3aacc9061..ce9f5bdd2 100644 --- a/packages/devlake/base/values.yaml +++ b/packages/devlake/base/values.yaml @@ -6,8 +6,8 @@ grafana: mysql: useExternal: true - externalServer: "${EXTERNAL_SERVER}" - externalPort: "${EXTERNAL_PORT}" + externalServer: "devlake-mysql-service" + externalPort: "3306" option: database: mysql diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 1a4834c3a..5f99540c3 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -3,6 +3,6 @@ resources: - mysql-init-job.yaml - rds-mysql.yaml -patchesStrategicMerge: - - patches/devlake-patch-sa.yaml +# patchesStrategicMerge: +# - patches/devlake-patch-sa.yaml diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml index 6f13bd6cd..e71bc0f5d 100644 --- a/packages/devlake/dev/mysql-init-job.yaml +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -1,8 +1,8 @@ +--- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: name: devlake-mysql-access-binding - namespace: devlake subjects: - kind: ServiceAccount name: devlake-mysql-access @@ -19,7 +19,7 @@ metadata: spec: template: metadata: - name: mysql-user-setup + generateName: mysql-user-setup- spec: serviceAccountName: devlake-mysql-access restartPolicy: OnFailure @@ -30,19 +30,15 @@ spec: - /bin/bash - -c - | + endpoint=$(kubectl get secret devlake-mysql-db-cluster-connection -n vela-system -o jsonpath='{.data.endpoint}' | base64 --decode) + kubectl create service externalname devlake-mysql-service --external-name=$endpoint -n devlake --dry-run=client -o yaml | kubectl apply -f - + password=$(kubectl get secret devlake-mysql-db-cluster-password -n vela-system -o jsonpath='{.data.password}' | base64 --decode) kubectl create secret generic devlake-mysql-auth \ --from-literal=MYSQL_USER=merico \ --from-literal=MYSQL_PASSWORD=merico \ --from-literal=MYSQL_DATABASE=lake \ --from-literal=MYSQL_ROOT_PASSWORD=$password \ - --from-literal=DB_URL=$endpoint - envFrom: - - secretRef: - name: devlake-mysql-db-cluster-password - namespace: vela-system - - secretRef: - name: devlake-mysql-db-cluster-connection - namespace: vela-system + --from-literal=DB_URL=mysql://merico:merico@devlake-mysql-service:3306/lake?charset=utf8mb4&parseTime=True containers: - name: create-grafana-user image: mysql:8 @@ -77,23 +73,4 @@ spec: envFrom: - secretRef: name: devlake-mysql-auth - - name: create-secret - image: bitnami/kubectl - command: - - /bin/bash - - -c - - | - kubectl create secret generic devlake-mysql-auth \ - --from-literal=MYSQL_USER=merico \ - --from-literal=MYSQL_PASSWORD=merico \ - --from-literal=MYSQL_DATABASE=lake \ - --from-literal=MYSQL_ROOT_PASSWORD=$password \ - --from-literal=DB_URL=$url - envFrom: - - secretRef: - name: devlake-mysql-db-cluster-password - namespace: vela-system - - secretRef: - name: devlake-mysql-db-cluster-connection - namespace: vela-system diff --git a/platform/backstage/templates/catalog-info.yaml b/platform/backstage/templates/catalog-info.yaml index a4f212c54..9e3a6d263 100644 --- a/platform/backstage/templates/catalog-info.yaml +++ b/platform/backstage/templates/catalog-info.yaml @@ -35,4 +35,4 @@ metadata: description: Holds system information i.e, hostname, IP, OS, etc spec: owner: guest - hostname: HOSTNAME \ No newline at end of file + hostname: HOSTNAME diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data-linux.sh b/platform/backstage/templates/devlake-onboarding/provisioner/generate-data-linux.sh similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data-linux.sh rename to platform/backstage/templates/devlake-onboarding/provisioner/generate-data-linux.sh diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data.sh b/platform/backstage/templates/devlake-onboarding/provisioner/generate-data.sh similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/generate-data.sh rename to platform/backstage/templates/devlake-onboarding/provisioner/generate-data.sh diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/init.sh b/platform/backstage/templates/devlake-onboarding/provisioner/init.sh similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/init.sh rename to platform/backstage/templates/devlake-onboarding/provisioner/init.sh diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/rbac.yaml b/platform/backstage/templates/devlake-onboarding/provisioner/rbac.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/rbac.yaml rename to platform/backstage/templates/devlake-onboarding/provisioner/rbac.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml b/platform/backstage/templates/devlake-onboarding/provisioner/wf-run.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-run.yaml rename to platform/backstage/templates/devlake-onboarding/provisioner/wf-run.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/provisioner/wf-templates.yaml similarity index 99% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml rename to platform/backstage/templates/devlake-onboarding/provisioner/wf-templates.yaml index 5232e6057..e4c003ff1 100644 --- a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/provisioner/wf-templates.yaml +++ b/platform/backstage/templates/devlake-onboarding/provisioner/wf-templates.yaml @@ -204,7 +204,7 @@ spec: git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git git pull --rebase if [ -f manifests/cicd-pipeline.yaml ]; then - yq -i '.spec.triggers[0].template.argoWorkflow.source.resource.spec.templates[0].steps += [[{ + yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.templates | select(.kind == "Sensor") | .steps += [[{ "name": "send-deployment-status", "templateRef": { "name": "cicd-deployments-dora-processing-template", diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/catalog-info.yaml b/platform/backstage/templates/devlake-onboarding/skeleton/catalog-info.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/catalog-info.yaml rename to platform/backstage/templates/devlake-onboarding/skeleton/catalog-info.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml b/platform/backstage/templates/devlake-onboarding/skeleton/manifests/events.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/events.yaml rename to platform/backstage/templates/devlake-onboarding/skeleton/manifests/events.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/devlake-onboarding/skeleton/manifests/exit-handler.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/exit-handler.yaml rename to platform/backstage/templates/devlake-onboarding/skeleton/manifests/exit-handler.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rbac.yaml b/platform/backstage/templates/devlake-onboarding/skeleton/manifests/rbac.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/rbac.yaml rename to platform/backstage/templates/devlake-onboarding/skeleton/manifests/rbac.yaml diff --git a/platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/devlake-onboarding/skeleton/manifests/wf-templates.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/templates-devlake-onboarding/skeleton/manifests/wf-templates.yaml rename to platform/backstage/templates/devlake-onboarding/skeleton/manifests/wf-templates.yaml diff --git a/platform/infra/terraform/bootstrap/main.tf b/platform/infra/terraform/bootstrap/main.tf index 7b738c706..6ab034d97 100644 --- a/platform/infra/terraform/bootstrap/main.tf +++ b/platform/infra/terraform/bootstrap/main.tf @@ -106,7 +106,7 @@ module "managed_grafana" { from_port = 3306 to_port = 3306 protocol = "tcp" - cidr_blocks = var.vpc_cidr + cidr_blocks = [var.vpc_cidr] } } diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf index 208e06d9c..94a84537b 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf @@ -7,8 +7,8 @@ module "vpc" { azs = local.azs private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] - database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 8)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k + 4)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k + 8)] enable_nat_gateway = true single_nat_gateway = true diff --git a/platform/traits/component-policy.cue b/platform/traits/component-policy.cue index 7ed7881d5..707ae2ac2 100644 --- a/platform/traits/component-policy.cue +++ b/platform/traits/component-policy.cue @@ -48,7 +48,7 @@ template: { spec: { name: "\(context.appName)-\(context.name)-iam-policy" forProvider: { - policy: "\(policy)" + policy: "\(parameter.policy)" } } } From a554bb834b76083bbd65e73a73a20caf1ad5c83e Mon Sep 17 00:00:00 2001 From: Zach Jacobson <72418868+zjaco13@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:41:29 -0700 Subject: [PATCH 26/63] testing for dora backstage fixes move names to dora and more fixes fix yq fix catalog info --- .../components/rds-cluster-mysql.yaml | 25 +++++- packages/devlake/dev/rds-mysql.yaml | 2 + packages/grafana/manifests/devlake.yaml | 10 +-- .../backstage/templates/catalog-info.yaml | 4 +- .../cicd-pipeline/provisioner/wf-run.yaml | 8 ++ .../skeleton/manifests/cicd-pipeline.yaml | 9 +- .../skeleton/manifests/wf-templates.yaml | 2 +- .../cicd-pipeline/template-cicd-pipeline.yaml | 6 ++ .../devlake-onboarding/provisioner/init.sh | 83 ------------------- .../provisioner/generate-data-linux.sh | 0 .../provisioner/generate-data.sh | 0 .../provisioner/rbac.yaml | 6 +- .../provisioner/wf-run.yaml | 14 ++-- .../provisioner/wf-templates.yaml | 40 ++++----- .../skeleton/catalog-info.yaml | 14 ++-- .../skeleton/manifests/events.yaml | 75 +++++++++++++---- .../skeleton/manifests/exit-handler.yaml | 4 +- .../skeleton/manifests/rbac.yaml | 6 +- .../skeleton/manifests/wf-templates.yaml | 0 .../template-dora-onboarding.yaml} | 14 ++-- platform/components/rds-cluster-mysql.cue | 38 +++++++-- platform/infra/terraform/bootstrap/main.tf | 8 ++ 22 files changed, 204 insertions(+), 164 deletions(-) delete mode 100644 platform/backstage/templates/devlake-onboarding/provisioner/init.sh rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/provisioner/generate-data-linux.sh (100%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/provisioner/generate-data.sh (100%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/provisioner/rbac.yaml (69%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/provisioner/wf-run.yaml (74%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/provisioner/wf-templates.yaml (92%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/skeleton/catalog-info.yaml (65%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/skeleton/manifests/events.yaml (63%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/skeleton/manifests/exit-handler.yaml (97%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/skeleton/manifests/rbac.yaml (71%) rename platform/backstage/templates/{devlake-onboarding => dora-onboarding}/skeleton/manifests/wf-templates.yaml (100%) rename platform/backstage/templates/{devlake-onboarding/template-devlake-onboarding.yaml => dora-onboarding/template-dora-onboarding.yaml} (90%) diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml index 5674bfb3d..81c52b6b8 100644 --- a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -53,14 +53,37 @@ spec: clusterIdentifierRef: name: "\(context.name)-cluster" engine: "aurora-mysql" instanceClass: "db.r5.large" + vpcSecurityGroupIDRefs: [{ + name: "\(context.name)-securitygroup" + }] + } + } + "\(context.name)-securitygroup": { + apiVersion: "ec2.aws.crossplane.io/v1beta1" + kind: "SecurityGroup" + metadata: name: "\(context.name)-securitygroup" + spec: forProvider: { + description: "SG for traffic to RDS from VPC" + groupName: "\(context.name)-securitygroup" + region: "\(parameter.region)" + ingress: [{ + fromPort: 3306 + ipProtocol: tcp + ipRanges: [{ + cidrIp: "\(parameter.vpcCidr)" + }] + toPort: 3306 + }] + vpcId: "\(parameter.vpcId)" } } } - parameter: { region: string subnetIds: [...string] + vpcId: string + vpcCidr: string } workload: definition: diff --git a/packages/devlake/dev/rds-mysql.yaml b/packages/devlake/dev/rds-mysql.yaml index 16c3cd6d1..3f2d01a2b 100644 --- a/packages/devlake/dev/rds-mysql.yaml +++ b/packages/devlake/dev/rds-mysql.yaml @@ -13,6 +13,8 @@ spec: - subnet-xxxxxx1 - subnet-xxxxxx2 - subnet-xxxxxx3 + vpcId: vpc-xxxxx + vpcCidr: 10.0.0.0/16 traits: - type: component-iam-policy properties: diff --git a/packages/grafana/manifests/devlake.yaml b/packages/grafana/manifests/devlake.yaml index d996e7506..d5fe406f1 100644 --- a/packages/grafana/manifests/devlake.yaml +++ b/packages/grafana/manifests/devlake.yaml @@ -9,7 +9,7 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORA.json" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/refs/heads/main/grafana/dashboards/DORA.json" --- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDashboard @@ -21,7 +21,7 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-DeploymentFrequency.json" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/refs/heads/main/grafana/dashboards/DORADetails-DeploymentFrequency.json" --- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDashboard @@ -33,7 +33,7 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-ChangeFailureRate.json" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/refs/heads/main/grafana/dashboards/DORADetails-ChangeFailureRate.json" --- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDashboard @@ -45,7 +45,7 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/refs/heads/main/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json" --- apiVersion: grafana.integreatly.org/v1beta1 kind: GrafanaDashboard @@ -57,4 +57,4 @@ spec: instanceSelector: matchLabels: dashboards: "external-grafana" - url: "https://raw.githubusercontent.com/apache/incubator-devlake/head/refs/main/grafana/dashboards/DORADetails-LeadTimeForChanges.json" + url: "https://raw.githubusercontent.com/apache/incubator-devlake/refs/heads/main/grafana/dashboards/DORADetails-LeadTimeforChanges.json" diff --git a/platform/backstage/templates/catalog-info.yaml b/platform/backstage/templates/catalog-info.yaml index 9e3a6d263..d56158a2e 100644 --- a/platform/backstage/templates/catalog-info.yaml +++ b/platform/backstage/templates/catalog-info.yaml @@ -22,7 +22,7 @@ spec: - ./create-dev-and-prod-env/template-create-dev-and-prod-env.yaml - ./cicd-pipeline/template-cicd-pipeline.yaml - ./create-env-aurora-postgres/template-env-aurora-postgres.yaml - - ./devlake-onboarding/template-devlake-onboarding.yaml + - ./dora-onboarding/template-dora-onboarding.yaml - ./s3-lambda-terraform/template-s3-lambda-terraform.yaml - ./stepfunctions-bedrock-terraform/template-stepfunctions-bedrock-terraform.yaml - ./apigw-sqs-terraform/template-apigw-sqs-terraform.yaml @@ -35,4 +35,4 @@ metadata: description: Holds system information i.e, hostname, IP, OS, etc spec: owner: guest - hostname: HOSTNAME + hostname: modern-engg-d773ae6fd4aebb9d.elb.us-west-2.amazonaws.com diff --git a/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml b/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml index df566882f..95f345c36 100644 --- a/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml +++ b/platform/backstage/templates/cicd-pipeline/provisioner/wf-run.yaml @@ -30,6 +30,10 @@ spec: value: ${{values.hostname}} - name: apprepo value: ${{values.hostname}}/gitea/giteaAdmin/${{values.appname}} + - name: branch + value: main + - name: tag-version + value: init - name: dockerfile_path value: ${{values.dockerfile_path}} - name: deployment_path @@ -65,6 +69,10 @@ spec: value: "{{workflow.parameters.apprepo}}" - name: aws_region value: "{{workflow.parameters.aws_region}}" + - name: branch + value: "{{workflow.parameters.branch}}" + - name: tag-version + value: "{{workflow.parameters.tag-version}}" - name: dockerfile_path value: "{{workflow.parameters.dockerfile_path}}" - name: deployment_path diff --git a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml index 534536780..22ead44a6 100644 --- a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml @@ -55,9 +55,9 @@ spec: value: 0.0.1 - name: aws_region value: ${{values.aws_region}} - - name: dockerfile-path + - name: dockerfile_path value: ${{values.dockerfile_path}} - - name: deployment-path + - name: deployment_path value: ${{values.deployment_path}} - name: appname value: ${{values.appname}} @@ -75,6 +75,7 @@ spec: requests: storage: 256Mi templates: + - name: run-cicd steps: - - name: run-cicd templateRef: @@ -87,7 +88,7 @@ spec: - name: apprepo value: "{{workflow.parameters.apprepo}}" - name: tag-version - value: "{{workflow.parameters.tag-version" + value: "{{workflow.parameters.tag-version}}" - name: branch value: "{{workflow.parameters.branch}}" - name: aws_region @@ -107,7 +108,7 @@ spec: dest: spec.arguments.parameters.7.value - src: dependencyName: source-dep - dataTemplate: "{{ .Input.headers }}" + dataTemplate: "{{ .Input.header }}" dest: spec.arguments.parameters.8.value --- apiVersion: v1 diff --git a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml index 17c1fab12..a73943902 100644 --- a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/wf-templates.yaml @@ -183,7 +183,7 @@ spec: value: "{{workflow.parameters.branch}}" - name: deployment-path value: "{{workflow.parameters.deployment_path}}" - depends: "warmup" + depends: "build-image" - name: cleandir container: diff --git a/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml b/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml index 105f6c272..b0f0ed0af 100644 --- a/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml +++ b/platform/backstage/templates/cicd-pipeline/template-cicd-pipeline.yaml @@ -92,6 +92,12 @@ spec: repoUrl: http://my-gitea-http.gitea.svc.cluster.local:3000/giteaAdmin/${{parameters.appname}}-cicd path: "manifests" + - action: debug:wait + name: Wait for ArgoCD + input: + seconds: 15 + + - action: cnoe:kubernetes:apply name: Run Workflow id: run-wf diff --git a/platform/backstage/templates/devlake-onboarding/provisioner/init.sh b/platform/backstage/templates/devlake-onboarding/provisioner/init.sh deleted file mode 100644 index bc224cca2..000000000 --- a/platform/backstage/templates/devlake-onboarding/provisioner/init.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -if [ $# -lt 2 ]; then - echo "Usage: $0 " - echo "Example: $0 http://localhost:4000 rust" - exit 1 -fi - -BASE_URL=$1 -curl -X GET ${BASE_URL}/proceed-db-migration - -projectName=$2 - -projectCreateRequest=$( - cat </dev/null 2>&1 BASE_URL="{{inputs.parameters.url}}" - curl -X GET ${BASE_URL}/proceed-db-migration + resp=$(curl -X GET ${BASE_URL}/proceed-db-migration) projectName="{{inputs.parameters.appname}}" @@ -140,8 +140,8 @@ spec: command: ["/bin/bash", "-c"] args: - | - TOKEN=$(echo "{{inputs.parameters.token-and-id}} | cut -d'|' -f1)" - ID=$(echo "{{inputs.parameters.token-and-id}} | cut -d'|' -f2)" + TOKEN=$(echo "{{inputs.parameters.token-and-id}}" | cut -d'|' -f1) + ID=$(echo "{{inputs.parameters.token-and-id}}" | cut -d'|' -f2) kubectl delete secret devlake-webhook-secret --ignore-not-found -n "{{workflow.parameters.namespace}}" kubectl create secret generic devlake-webhook-secret -n "{{workflow.parameters.namespace}}" \ --from-literal=DEVLAKE_TOKEN="$TOKEN" @@ -161,7 +161,7 @@ spec: - | apk add curl jq hooks_api_url="{{workflow.parameters.hostname}}"/gitea/api/v1/repos/giteaAdmin/"{{workflow.parameters.appname}}"/hooks - webhook_url="http://{{workflow.parameters.url}}"/argo-events/"{{workflow.parameters.appname}}-dora/incidents" + webhook_url="http://{{workflow.parameters.hostname}}"/argo-events/"{{workflow.parameters.appname}}-dora/incidents" webhook_exists=$(curl -k -X 'GET' "${hooks_api_url}" -H "accept: application/json" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" | jq --arg url "$webhook_url" 'map(select(.config.url == $url)) | length > 0') if ! $webhook_exists; then echo "Webhook is not found, creating webhook with $webhook_url" @@ -170,7 +170,7 @@ spec: echo "Webhook with url $webhook_url already exists" fi - webhook_url="http://{{workflow.parameters.url}}"/argo-events/"{{workflow.parameters.appname}}-dora/pull-requests" + webhook_url="http://{{workflow.parameters.hostname}}"/argo-events/"{{workflow.parameters.appname}}-dora/pull-requests" webhook_exists=$(curl -k -X 'GET' "${hooks_api_url}" -H "accept: application/json" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" | jq --arg url "$webhook_url" 'map(select(.config.url == $url)) | length > 0') if ! $webhook_exists; then echo "Webhook is not found, creating webhook with $webhook_url" @@ -183,7 +183,6 @@ spec: inputs: parameters: - name: apprepo - - name: exit-hook-name container: envFrom: - secretRef: @@ -199,12 +198,15 @@ spec: - | apk add --no-cache yq set -ex + mkdir ./src + git clone https://"{{inputs.parameters.apprepo}}".git -c http.sslVerify=false ./src + cd ./src git config --global user.email "noreply@aws.com" git config --global user.name ARGO git remote set-url origin https://giteaAdmin:${GITEA_TOKEN}@"{{inputs.parameters.apprepo}}".git git pull --rebase if [ -f manifests/cicd-pipeline.yaml ]; then - yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.templates | select(.kind == "Sensor") | .steps += [[{ + yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.templates[0].steps += [[{ "name": "send-deployment-status", "templateRef": { "name": "cicd-deployments-dora-processing-template", @@ -214,11 +216,11 @@ spec: "parameters": [ { "name": "body", - "value": "{{workflow.parameters.body}}" + "value": {{"'{{workflow.parameters.body}}'"}} }, { "name": "headers", - "value": "{{workflow.parameters.headers}}" + "value": {{"'{{workflow.parameters.headers}}'"}} } ] } @@ -237,11 +239,11 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: devlake-data-gen-template + name: dora-data-gen-template namespace: ${{values.namespace}} spec: templates: - - name: devlake-data-gen + - name: dora-data-gen dag: tasks: - name: data-gen @@ -250,7 +252,7 @@ spec: - name: devlake-generate-data container: image: debian:12 - command: [/bin/bash] + command: [/bin/bash, -c] envFrom: - secretRef: name: devlake-webhook-secret diff --git a/platform/backstage/templates/devlake-onboarding/skeleton/catalog-info.yaml b/platform/backstage/templates/dora-onboarding/skeleton/catalog-info.yaml similarity index 65% rename from platform/backstage/templates/devlake-onboarding/skeleton/catalog-info.yaml rename to platform/backstage/templates/dora-onboarding/skeleton/catalog-info.yaml index b54ccd247..5a7b0e87e 100644 --- a/platform/backstage/templates/devlake-onboarding/skeleton/catalog-info.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/catalog-info.yaml @@ -1,17 +1,17 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: ${{values.appname}}-devlake - description: Onboard ${{ values.appname }} team to DevLake + name: ${{values.appname}}-dora + description: Onboard ${{ values.appname }} team to DORA monitoring annotations: backstage.io/kubernetes-namespace: ${{values.namespace}} - backstage.io/kubernetes-label-selector: 'entity-id=${{values.appname}}-devlake' - argocd/app-selector: 'entity-id=${{values.appname}}-devlake' + backstage.io/kubernetes-label-selector: 'entity-id=${{values.appname}}-dora' + argocd/app-selector: 'entity-id=${{values.appname}}-dora' argo-workflows.cnoe.io/cluster-name: local - argo-workflows.cnoe.io/namespace: team-${{values.appname}} - argo-workflows.cnoe.io/label-selector: 'entity-id=${{values.appname}}-devlake' + argo-workflows.cnoe.io/namespace: ${{values.namespace}} + argo-workflows.cnoe.io/label-selector: 'entity-id=${{values.appname}}-dora' links: - - url: https://${{values.hostname}}/gitea?repo=${{values.appname}}-devlake + - url: https://${{values.hostname}}/gitea?repo=${{values.appname}}-dora title: Repo URL icon: github spec: diff --git a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/events.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml similarity index 63% rename from platform/backstage/templates/devlake-onboarding/skeleton/manifests/events.yaml rename to platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml index a7c03c750..a99f491a1 100644 --- a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/events.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml @@ -1,7 +1,8 @@ apiVersion: v1 kind: Service metadata: - name: devlake-webhook-svc + name: ${{values.appname}}-dora-webhook-svc + namespace: ${{values.namespace}} spec: type: ClusterIP # Or ClusterIP if using Ingress ports: @@ -11,15 +12,16 @@ spec: #authSecret: #name: selector: - eventsource-name: gitea-devlake-webhook + eventsource-name: ${{values.appname}}-gitea-dora-webhook --- apiVersion: argoproj.io/v1alpha1 kind: EventSource metadata: - name: gitea-devlake-webhook + name: ${{values.appname}}-gitea-dora-webhook + namespace: ${{values.namespace}} labels: - eventsource-name: gitea-devlake-webhook + eventsource-name: ${{values.appname}}-gitea-dora-webhook spec: service: ports: @@ -43,10 +45,11 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Sensor metadata: - name: gitea-webhook-sensor + name: ${{values.appname}}-gitea-webhook-sensor + namespace: ${{values.namespace}} spec: template: - serviceAccountName: wf-sa + serviceAccountName: ${{values.appname}}-dora-webhook-sa dependencies: - name: incidents-webhook eventSourceName: gitea-webhook @@ -68,9 +71,9 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: incidents-workflow- + generateName: ${{values.appname}}-dora-incidents-workflow- spec: - serviceAccountName: devlake-webhook-sa + serviceAccountName: ${{values.appname}}-dora-webhook-sa entrypoint: process-incident workflowTemplateRef: name: incidents-workflow-template @@ -79,7 +82,7 @@ spec: - name: body value: '{{.Input.body}}' - name: headers - value: '{{.Input.headers}}' + value: '{{.Input.header}}' - name: url value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/issues parameters: @@ -87,6 +90,10 @@ spec: dependencyName: incidents-webhook dataKey: body dest: spec.arguments.parameters.0.value + - src: + dependencyName: incidents-webhook + dataKey: header + dest: spec.arguments.parameters.1.value - template: name: deployments-workflow-trigger @@ -98,8 +105,9 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: deployments-workflow- + generateName: ${{values.appname}}-dora-deployments-workflow- spec: + serviceAccountName: ${{values.appname}}-dora-webhook-sa workflowTemplateRef: name: deployments-workflow-template arguments: @@ -107,7 +115,7 @@ spec: - name: body value: '{{.Input.body}}' - name: headers - value: '{{.Input.headers}}' + value: '{{.Input.header}}' - name: url value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/deployments parameters: @@ -117,7 +125,7 @@ spec: dest: spec.arguments.parameters.0.value - src: dependencyName: deployments-webhook - dataKey: headers + dataKey: header dest: spec.arguments.parameters.1.value - template: @@ -130,8 +138,9 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: pull-requests-workflow- + generateName: ${{values.appname}}-dora-pull-requests-workflow- spec: + serviceAccountName: ${{values.appname}}-dora-webhook-sa workflowTemplateRef: name: pull-requests-workflow-template entrypoint: process-pull-request @@ -140,7 +149,7 @@ spec: - name: body value: '{{.Input.body}}' - name: headers - value: '{{.Input.headers}}' + value: '{{.Input.header}}' - name: url value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/pullrequests parameters: @@ -148,3 +157,41 @@ spec: dependencyName: pull-requests-webhook dataKey: body dest: spec.arguments.parameters.0.value + - src: + dependencyName: pull-requests-webhook + dataKey: header + dest: spec.arguments.parameters.1.value +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${{values.appname}}-dora-webhook-ing + namespace: ${{values.namespace}} + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: 512m + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: ${{values.hostname}} + http: + paths: + - backend: + service: + name: ${{values.appname}}-dora-webhook-svc + port: + number: 12000 + path: /argo-events/${{values.appname}}-dora(/|$)(.*) + pathType: ImplementationSpecific + - host: localhost + http: + paths: + - backend: + service: + name: ${{values.appname}}-dora-webhook-svc + port: + number: 12000 + path: /argo-events/${{values.appname}}-dora(/|$)(.*) + pathType: ImplementationSpecific + diff --git a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml similarity index 97% rename from platform/backstage/templates/devlake-onboarding/skeleton/manifests/exit-handler.yaml rename to platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml index a1a5c572f..4b2f8e23e 100644 --- a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/exit-handler.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml @@ -1,11 +1,11 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: cicd-deployments-dora-processing-template + name: ${{values.appname}}-cicd-deployments-dora-processing-template namespace: ${{values.namespace}} spec: entrypoint: handle-rollout-check-and-process-deploy - serviceAccountName: devlake-webhook-sa + serviceAccountName: ${{values.appname}}-dora-webhook-sa volumes: - name: argocd-cluster-secret secret: diff --git a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/rbac.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml similarity index 71% rename from platform/backstage/templates/devlake-onboarding/skeleton/manifests/rbac.yaml rename to platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml index 6c12dba4f..01dbb7e8c 100644 --- a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/rbac.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml @@ -1,17 +1,17 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: devlake-webhook-sa + name: ${{values.appname}}-dora-webhook-sa namespace: ${{values.namespace}} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: devlake-webhook-binding + name: ${{values.appname}}-dora-webhook-binding namespace: ${{values.namespace}} subjects: - kind: ServiceAccount - name: devlake-webhook-sa + name: ${{values.appname}}-dora-webhook-sa namespace: ${{values.namespace}} roleRef: kind: ClusterRole diff --git a/platform/backstage/templates/devlake-onboarding/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/wf-templates.yaml similarity index 100% rename from platform/backstage/templates/devlake-onboarding/skeleton/manifests/wf-templates.yaml rename to platform/backstage/templates/dora-onboarding/skeleton/manifests/wf-templates.yaml diff --git a/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml b/platform/backstage/templates/dora-onboarding/template-dora-onboarding.yaml similarity index 90% rename from platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml rename to platform/backstage/templates/dora-onboarding/template-dora-onboarding.yaml index c84392274..8a3dd243a 100644 --- a/platform/backstage/templates/devlake-onboarding/template-devlake-onboarding.yaml +++ b/platform/backstage/templates/dora-onboarding/template-dora-onboarding.yaml @@ -1,9 +1,9 @@ apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: - description: Onboard an Application Team to DevLake - name: devlake-onboarding - title: Onboard App to DevLake + description: Onboard an Application Team to DORA monitoring + name: dora-onboarding + title: Onboard App to DORA spec: owner: guest type: service @@ -34,9 +34,9 @@ spec: name: Publishing to a gitea git repository action: publish:gitea input: - description: Devlake onboarding template + description: DORA onboarding template # Hard coded value for this demo purposes only. - repoUrl: ${{ steps['fetchSystem'].output.entity.spec.hostname }}/gitea?repo=${{parameters.appname}}-devlake + repoUrl: ${{ steps['fetchSystem'].output.entity.spec.hostname }}/gitea?repo=${{parameters.appname}}-dora defaultBranch: main - action: fetch:template @@ -77,12 +77,12 @@ spec: name: Create ArgoCD App action: cnoe:create-argocd-app input: - appName: ${{parameters.appname}} + appName: ${{parameters.appname}}-dora appNamespace: team-${{ parameters.appname }} argoInstance: in-cluster projectName: default # necessary until we generate our own cert - repoUrl: http://my-gitea-http.gitea.svc.cluster.local:3000/giteaAdmin/${{parameters.appname}}-devlake + repoUrl: http://my-gitea-http.gitea.svc.cluster.local:3000/giteaAdmin/${{parameters.appname}}-dora path: "manifests" - id: register name: Register diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue index c89376eef..3accae9c5 100644 --- a/platform/components/rds-cluster-mysql.cue +++ b/platform/components/rds-cluster-mysql.cue @@ -60,14 +60,40 @@ template: { clusterIdentifierRef: name: "\(context.name)-cluster" engine: "aurora-mysql" instanceClass: "db.r5.large" + vpcSecurityGroupIDRefs: [{ + name: "\(context.name)-securitygroup" + }] } } } + "\(context.name)-securitygroup": { + apiVersion: "ec2.aws.crossplane.io/v1beta1" + kind: "SecurityGroup" + metadata: name: "\(context.name)-securitygroup" + spec: { + forProvider: { + description: "SG for traffic to RDS from VPC" + groupName: "\(context.name)-securitygroup" + region: "\(parameter.region)" + ingress: [{ + fromPort: 3306 + ipProtocol: tcp + ipRanges: [{ + cidrIp: "\(parameter.vpcCidr)" + }] + toPort: 3306 + }] + vpcId: "\(parameter.vpcId)" + } + } + } - } - - parameter: { - region: string - subnetIds: [...string] - } + + } + parameter: { + region: string + subnetIds: [...string] + vpcId: string + vpcCidr: string + } } diff --git a/platform/infra/terraform/bootstrap/main.tf b/platform/infra/terraform/bootstrap/main.tf index 6ab034d97..e38e35848 100644 --- a/platform/infra/terraform/bootstrap/main.tf +++ b/platform/infra/terraform/bootstrap/main.tf @@ -108,6 +108,14 @@ module "managed_grafana" { protocol = "tcp" cidr_blocks = [var.vpc_cidr] } + + egress_http = { + description = "Allow egress to http" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks= ["0.0.0.0/0"] + } } vpc_configuration = { From 510e7c56d3c5f8090e98e359082a9e4538fd02f8 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 10 Apr 2025 17:51:52 -0700 Subject: [PATCH 27/63] changes to autogen rds --- packages/devlake/dev/kustomization.yaml | 40 ++++++++++++- packages/devlake/dev/rds-mysql.yaml | 8 +-- packages/grafana/manifests/mysql.yaml | 2 +- .../provisioner/wf-templates.yaml | 31 +++++++--- .../skeleton/manifests/events.yaml | 12 ++-- .../skeleton/manifests/exit-handler.yaml | 56 ++++++++++++++----- .../skeleton/manifests/rbac.yaml | 3 +- .../skeleton/manifests/wf-templates.yaml | 3 +- .../grafana-workload-dashboards.yaml | 2 +- .../terraform/scripts/devlake/devlake-init.sh | 37 ++++++++++++ .../scripts/devlake}/generate-data-linux.sh | 2 +- .../scripts/devlake/generate-data-mac.sh} | 0 .../devlake/{init.sh => project-init.sh} | 23 +++----- .../infra/terraform/setup-environments.sh | 23 ++++++-- 14 files changed, 180 insertions(+), 62 deletions(-) create mode 100755 platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh rename platform/{backstage/templates/dora-onboarding/provisioner => infra/terraform/mgmt/terraform/scripts/devlake}/generate-data-linux.sh (99%) mode change 100644 => 100755 rename platform/{backstage/templates/dora-onboarding/provisioner/generate-data.sh => infra/terraform/mgmt/terraform/scripts/devlake/generate-data-mac.sh} (100%) rename platform/infra/terraform/mgmt/terraform/scripts/devlake/{init.sh => project-init.sh} (92%) diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 5f99540c3..22ad11a0e 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -3,6 +3,40 @@ resources: - mysql-init-job.yaml - rds-mysql.yaml -# patchesStrategicMerge: -# - patches/devlake-patch-sa.yaml - +patchesStrategicMerge: + - patches/devlake-patch-sa.yaml + +configMapGenerator: +- name: vpc-config + envs: + - infrastructure-values.env + +vars: +- name: SUBNET_ID_1 + objref: + kind: ConfigMap + name: vpc-config + apiVersion: v1 + fieldref: + fieldpath: data.subnet_id_1 +- name: SUBNET_ID_2 + objref: + kind: ConfigMap + name: vpc-config + apiVersion: v1 + fieldref: + fieldpath: data.subnet_id_2 +- name: SUBNET_ID_3 + objref: + kind: ConfigMap + name: vpc-config + apiVersion: v1 + fieldref: + fieldpath: data.subnet_id_3 +- name: VPC_ID + objref: + kind: ConfigMap + name: vpc-config + apiVersion: v1 + fieldref: + fieldpath: data.vpc_id diff --git a/packages/devlake/dev/rds-mysql.yaml b/packages/devlake/dev/rds-mysql.yaml index 3f2d01a2b..ec05493f6 100644 --- a/packages/devlake/dev/rds-mysql.yaml +++ b/packages/devlake/dev/rds-mysql.yaml @@ -10,10 +10,10 @@ spec: properties: region: us-west-2 subnetIds: - - subnet-xxxxxx1 - - subnet-xxxxxx2 - - subnet-xxxxxx3 - vpcId: vpc-xxxxx + - $(SUBNET_ID_1) + - $(SUBNET_ID_2) + - $(SUBNET_ID_3) + vpcId: $(VPC_ID) vpcCidr: 10.0.0.0/16 traits: - type: component-iam-policy diff --git a/packages/grafana/manifests/mysql.yaml b/packages/grafana/manifests/mysql.yaml index dff886ec3..fd874c790 100644 --- a/packages/grafana/manifests/mysql.yaml +++ b/packages/grafana/manifests/mysql.yaml @@ -5,7 +5,7 @@ metadata: name: grafana-mysql-endpoint # How to do programatically namespace: grafana-operator data: - MYSQL_ENDPOINT: ${NODEPORT_ENDPOINT} + MYSQL_ENDPOINT: ${MYSQL_ENDPOINT} --- apiVersion: v1 kind: Secret diff --git a/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml b/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml index 8056c4b21..4f5fabd0e 100644 --- a/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml +++ b/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml @@ -126,9 +126,8 @@ spec: resp=$(curl -X PATCH -H "Content-Type: application/json" ${BASE_URL}/blueprints/$blueprintID -d "$blueprintPatchRequest") - resp=$(curl -X POST -H "Content-Type: application/json" ${BASE_URL}/blueprints/${blueprintID}/trigger -d '{"skipCollectors":false,"fullSync":false}') - echo "$webhookApiKey|$webhookID" + echo "$webhookApiKey|$webhookID|$blueprintID" - name: create-devlake-webhook-secret-and-cm @@ -141,13 +140,18 @@ spec: args: - | TOKEN=$(echo "{{inputs.parameters.token-and-id}}" | cut -d'|' -f1) - ID=$(echo "{{inputs.parameters.token-and-id}}" | cut -d'|' -f2) + HOOK_ID=$(echo "{{inputs.parameters.token-and-id}}" | cut -d'|' -f2) + BP_ID=$(echo "{{inputs.parameters.token-and-id}}" | cut -d'|' -f3) kubectl delete secret devlake-webhook-secret --ignore-not-found -n "{{workflow.parameters.namespace}}" kubectl create secret generic devlake-webhook-secret -n "{{workflow.parameters.namespace}}" \ --from-literal=DEVLAKE_TOKEN="$TOKEN" kubectl delete configmap devlake-webhook-id --ignore-not-found -n "{{workflow.parameters.namespace}}" kubectl create configmap devlake-webhook-id -n "{{workflow.parameters.namespace}}" \ - --from-literal=DEVLAKE_HOOK_ID="$ID" + --from-literal=DEVLAKE_HOOK_ID="$HOOK_ID" --from-literal=DEVLAKE_BP_ID="$BP_ID" + + kubectl get secret dev-cluster-argo-secret -n argocd -o yaml | \ + sed 's/namespace: argocd/namespace: "{{workflow.parameters.namespace}}"/' | \ + kubectl create -f - - name: create-gitea-webhook @@ -209,22 +213,29 @@ spec: yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.templates[0].steps += [[{ "name": "send-deployment-status", "templateRef": { - "name": "cicd-deployments-dora-processing-template", + "name": "{{workflow.parameters.appname}}"-cicd-deployments-dora-processing-template", "template": "handle-rollout-check-and-process-deploy" }, "arguments": { "parameters": [ { "name": "body", - "value": {{"'{{workflow.parameters.body}}'"}} + "value": 'PLACEHOLDER_OPENworkflow.parameters.bodyPLACEHOLDER_CLOSE' }, { "name": "headers", - "value": {{"'{{workflow.parameters.headers}}'"}} + "value": 'PLACEHOLDER_OPENworkflow.parameters.headersPLACEHOLDER_CLOSE' } ] } }]]' manifests/cicd-pipeline.yaml + sed -e 's/PLACEHOLDER_OPEN/{{/g' -e 's/PLACEHOLDER_CLOSE/}}/g' manifests/cicd-pipeline.yaml > manifests/cicd-pipeline.yaml.tmp && mv manifests/cicd-pipeline.yaml.tmp manifests/cicd-pipeline.yaml + yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.volumes += [{ + "name": "argocd-cluster-secret", + "secret": { + "secretName": "dev-cluster-argo-secret", + } + }]' manifests/cicd-pipeline.yaml fi m_count=$(git status -s | wc -l) if [ $m_count -ge 1 ]; then @@ -263,8 +274,10 @@ spec: apt update apt install -y curl openssl coreutils BASE_URL="http://devlake-ui.devlake.svc.cluster.local:4000" + TRIGGER_URL="http://devlake-lake.devlake.svc.cluster.local:8080" API_KEY="$DEVLAKE_TOKEN" CONNECTION_ID="$DEVLAKE_HOOK_ID" + BLUEPRINT_ID="$DEVLAKE_BP_ID" rand() { @@ -439,4 +452,8 @@ spec: done echo "Data generation complete. Created $((pr_id - 1)) pull requests, $((deploy_id - 1)) deployments, and $((incident_id - 1)) incidents." + sleep 10 + + echo "Triggering DORA Metrics Collection" + resp=$(curl -s -X POST -H "Content-Type: application/json" "${TRIGGER_URL}/blueprints/${BLUEPRINT_ID}/trigger" -d '{"skipCollectors":false,"fullSync":false}') diff --git a/platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml index a99f491a1..d03a3671b 100644 --- a/platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/events.yaml @@ -52,13 +52,13 @@ spec: serviceAccountName: ${{values.appname}}-dora-webhook-sa dependencies: - name: incidents-webhook - eventSourceName: gitea-webhook + eventSourceName: ${{values.appname}}-gitea-dora-webhook eventName: incidents-webhook - name: deployments-webhook - eventSourceName: gitea-webhook + eventSourceName: ${{values.appname}}-gitea-dora-webhook eventName: deployments-webhook - name: pull-requests-webhook - eventSourceName: gitea-webhook + eventSourceName: ${{values.appname}}-gitea-dora-webhook eventName: pull-requests-webhook triggers: - template: @@ -84,7 +84,7 @@ spec: - name: headers value: '{{.Input.header}}' - name: url - value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/issues + value: http://devlake-ui.devlake.svc.cluster.local:4000 parameters: - src: dependencyName: incidents-webhook @@ -117,7 +117,7 @@ spec: - name: headers value: '{{.Input.header}}' - name: url - value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/deployments + value: http://devlake-ui.devlake.svc.cluster.local:4000 parameters: - src: dependencyName: deployments-webhook @@ -151,7 +151,7 @@ spec: - name: headers value: '{{.Input.header}}' - name: url - value: http://devlake-ui.devlake.svc.cluster.local:4000/api/rest/plugins/webhook/connections/1/pullrequests + value: http://devlake-ui.devlake.svc.cluster.local:4000 parameters: - src: dependencyName: pull-requests-webhook diff --git a/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml index 4b2f8e23e..c9dc6c0bb 100644 --- a/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml @@ -15,17 +15,24 @@ spec: parameters: - name: body - name: headers - - name: url - value: devlake-ui.devlake.svc.cluster.local:4000 - name: appname - value: ${{values.appName}} - - name: namespace - value: ${{values.appNamespace}} + value: ${{values.appname}} templates: - name: handle-rollout-check-and-process-deploy steps: + - - name: fetch-argo-plugin + template: download-argo-plugin + - - name: check-rollout template: check-rollout-status + arguments: + parameters: + - name: namespace + value: team-"{{workflow.parameters.appname}}" + artifacts: + - name: rollout_plugin + from: "{{steps.fetch-argo-plugin.outputs.artifacts.rollout_plugin}}" + - - name: process-and-send-deployment template: process-deployment arguments: @@ -35,7 +42,7 @@ spec: - name: body value: "{{workflow.parameters.body}}" - name: url - value: "{{workflow.parameters.url}}" + value: devlake-ui.devlake.svc.cluster.local:4000 - name: headers value: "{{workflow.parameters.headers}}" @@ -87,25 +94,44 @@ spec: curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" $webhook_url + - name: download-argo-plugin + container: + image: alpine/curl:8.12.1 + command: [/bin/bash, -c] + args: + - | + curl -LO https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64 + chmod +x ./kubectl-argo-rollouts-linux-amd64 + mv ./kubectl-argo-rollouts-linux-amd64 > /tmp/kubectl-argo-rollouts-linux-amd64 + outputs: + artifacts: + - name: rollout_plugin + path: /tmp/kubectl-argo-rollouts-linux-amd64 + + - name: check-rollout-status - script: + inputs: + parameters: + - name: namespace + artifacts: + - name: rollout_plugin + path: /tmp/kubectl-argo-rollouts-linux-amd64 + container: image: bitnami/kubectl:latest - command: [bash] + command: [/bin/bash, -c] volumeMounts: - name: argocd-cluster-secret mountPath: /tmp/argocd-cluster readOnly: true - source: | + args: + - | mkdir -p ~/.kube cp /tmp/argocd-cluster/config ~/.kube/config - # Install kubectl-argo-rollouts plugin - curl -LO https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64 - chmod +x ./kubectl-argo-rollouts-linux-amd64 - mv ./kubectl-argo-rollouts-linux-amd64 /usr/local/bin/kubectl-argo-rollouts + mv /tmp/kubectl-argo-rollouts-linux-amd64 /usr/local/bin/kubectl-argo-rollouts SEARCH_WORD="{{workflow.parameters.appname}}" - ROLLOUT_NAME=$(kubectl get rollout -n {{workflow.parameters.namespace}} -o json | \ + ROLLOUT_NAME=$(kubectl get rollout -n "{{inputs.parameters.namespace}}" -o json | \ jq -r --arg WORD "$SEARCH_WORD" '.items[] | select(.metadata.name | contains($WORD)) | .metadata.name') if [ -z "$ROLLOUT_NAME" ]; then @@ -117,7 +143,7 @@ spec: INTERVAL=10 # Check every 10 seconds ELAPSED=0 - if kubectl argo rollouts status $ROLLOUT_NAME -n "{{workflow.parameters.namespace}}" --timeout 2m; then + if kubectl argo rollouts status $ROLLOUT_NAME -n "{{inputs.parameters.namespace}}" --timeout 2m; then echo "SUCCESS" exit 0 else diff --git a/platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml index 01dbb7e8c..bc3f3ee72 100644 --- a/platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/rbac.yaml @@ -5,10 +5,9 @@ metadata: namespace: ${{values.namespace}} --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: name: ${{values.appname}}-dora-webhook-binding - namespace: ${{values.namespace}} subjects: - kind: ServiceAccount name: ${{values.appname}}-dora-webhook-sa diff --git a/platform/backstage/templates/dora-onboarding/skeleton/manifests/wf-templates.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/wf-templates.yaml index 7e31111b3..f086ea231 100644 --- a/platform/backstage/templates/dora-onboarding/skeleton/manifests/wf-templates.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/wf-templates.yaml @@ -83,6 +83,7 @@ spec: apk add --no-cache jq curl webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/${DEVLAKE_HOOK_ID}/pull_requests + echo '{{inputs.parameters.body}}' PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq ' { id: (.pull_request.id | tostring), @@ -97,7 +98,7 @@ spec: url: .pull_request.html_url, authorId: .pull_request.user.login, authorName: (.pull_request.user.name // .pull_request.user.login), - reviewers: [.pull_request.requested_reviewers[].login], + reviewers: (.pull_request.requested_reviewers // [] | map(.login)), comments: (.pull_request.comments // 0), commits: (.pull_request.commits // 0), additions: (.pull_request.additions // 0), diff --git a/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml b/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml index 8946ae021..5ba26fb27 100644 --- a/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml +++ b/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml @@ -17,7 +17,7 @@ spec: patch: |- - op: replace path: /data/MYSQL_ENDPOINT - value: ${DEVLAKE_MYSQL_NODEPORT} + value: ${DEVLAKE_MYSQL_ENDPOINT} destination: name: 'dev-cluster' namespace: grafana-operator diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh new file mode 100755 index 000000000..c080c7e02 --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +kubectl wait --for=jsonpath=.status.health.status=Healthy -n argocd application/devlake +kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=60s +kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=60s + +kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & +pid=$! +trap '{ + kill $pid +}' EXIT + +echo "waiting for port forward to be ready" +while ! nc -vz localhost 9090 >/dev/null 2>&1; do + sleep 2 +done + + +OS=$(uname -s) +STRINGS=("dotnet" "go" "java" "nextjs" "") +for str in "${STRINGS[@]}"; do +./project-init.sh $str + process_args "$MAC_SCRIPT" "$str" +done + +case "$OS" in + "Darwin") + ./generate-data-mac.sh + ;; + "Linux") + ./generate-data-linux.sh + ;; + *) + echo "Unsupported operating system: $OS" + exit 1 + ;; +esac diff --git a/platform/backstage/templates/dora-onboarding/provisioner/generate-data-linux.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/generate-data-linux.sh old mode 100644 new mode 100755 similarity index 99% rename from platform/backstage/templates/dora-onboarding/provisioner/generate-data-linux.sh rename to platform/infra/terraform/mgmt/terraform/scripts/devlake/generate-data-linux.sh index cb3c79c32..f7e00aa53 --- a/platform/backstage/templates/dora-onboarding/provisioner/generate-data-linux.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/generate-data-linux.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash if [ $# -lt 3 ]; then echo "Usage: $0 " diff --git a/platform/backstage/templates/dora-onboarding/provisioner/generate-data.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/generate-data-mac.sh similarity index 100% rename from platform/backstage/templates/dora-onboarding/provisioner/generate-data.sh rename to platform/infra/terraform/mgmt/terraform/scripts/devlake/generate-data-mac.sh diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh similarity index 92% rename from platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh rename to platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh index 962c29eb1..23f43e481 100755 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/init.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh @@ -1,23 +1,16 @@ -# do db migration, add some data to devlake -kubectl wait --for=jsonpath=.status.health.status=Healthy -n argocd application/devlake -kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=60s -kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=60s +#!/bin/bash -kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & -pid=$! -trap '{ - kill $pid -}' EXIT +if [ $# -lt 1 ]; then + echo "Usage: $0 " + echo "Example: $0 modengg" + exit 1 +fi -echo "waiting for port forward to be ready" -while ! nc -vz localhost 9090 >/dev/null 2>&1; do - sleep 2 -done echo "Performing DB Migration" curl -X GET localhost:9090/proceed-db-migration -projectName="modengg" +projectName=$1 projectCreateRequest=$( cat < ${REPO_ROOT}/packages/devlake/dev/infrastructure-values.env << EOF +vpc_id=$VPC_ID +subnet_id_1=${SUBNET_ARRAY[0]} +subnet_id_2=${SUBNET_ARRAY[1]} +subnet_id_3=${SUBNET_ARRAY[2]} +EOF + # Deploy the apps on IDP Builder and ArgoCD ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh @@ -65,6 +79,7 @@ export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_priva echo "private subnets are : " $TF_eks_cluster_private_subnets # For Database and EC2 database export TF_eks_cluster_vpc_cidr=$(terraform output -raw vpc_cidr) +export TF_eks_cluster_db_subnets=$(terraform output -raw database_subnet_ids) export TF_eks_cluster_private_az=$(terraform output -json availability_zones) echo "private az are : " $TF_eks_cluster_private_az export KEYCLOAK_NAMESPACE=keycloak @@ -291,9 +306,7 @@ terraform -chdir=post-deploy apply -var aws_region="${TF_VAR_aws_region}" \ -var dev_cluster_name="${TF_VAR_dev_cluster_name}" \ -var prod_cluster_name="${TF_VAR_prod_cluster_name}" -auto-approve -NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') -NODE_PORT=$(kubectl get service devlake-mysql -n devlake -o jsonpath='{.spec.ports[0].nodePort}') -export DEVLAKE_MYSQL_NODEPORT="${NODE_IP}:${NODE_PORT}" +export DEVLAKE_MYSQL_ENDPOINT=$(aws rds describe-db-clusters --region ${TF_VAR_aws_region} --db-cluster-identifier devlake-mysql-db-cluster --query 'DBClusters[0].ReaderEndpoint' --output text) # Setup Applications on Clusters using ArgoCD on the management cluster @@ -304,7 +317,7 @@ sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANC sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-comp-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-comp-prod.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-dev.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/crossplane-provider-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/crossplane-provider-prod.yml -sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" -e "s#\${DEVLAKE_MYSQL_NODEPORT}#${DEVLAKE_MYSQL_NODEPORT}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml +sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" -e "s#\${DEVLAKE_MYSQL_ENDPOINT}#${DEVLAKE_MYSQL_ENDPOINT}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/grafana-workload-dashboards.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/grafana-workload-dashboards.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-dev.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-dev.yml sed -e "s#\${GITHUB_URL}#${GITHUB_URL}#g" -e "s#\${GITHUB_BRANCH}#${GITHUB_BRANCH}#g" ${REPO_ROOT}/platform/infra/terraform/deploy-apps/kubevela-prod.yaml >${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/kubevela-prod.yml @@ -314,8 +327,6 @@ kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/deploy-apps/manifests/ # Setup Gitea Repo ${REPO_ROOT}/platform/infra/terraform/giteaInit.sh -# Add AMG URL to Devlake -kubectl set env deployment devlake-ui GRAFANA_ENDPOINT="https://$WORKSPACE_ENDPOINT/" -n devlake # Sleeping for Crossplane to be ready in DEV and PROD Cluster and restarting backstage pod kubectl rollout restart deployment backstage -n backstage sleep 120 From 5332041b33655a3c59cb62379420a21dd4ae9ca5 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 14 Apr 2025 19:58:09 -0700 Subject: [PATCH 28/63] fix mysql component --- .../templates/components/rds-cluster-mysql.yaml | 2 +- platform/components/rds-cluster-mysql.cue | 2 +- platform/infra/terraform/setup-environments.sh | 16 +++++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml index 81c52b6b8..f5059e69f 100644 --- a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -68,7 +68,7 @@ spec: region: "\(parameter.region)" ingress: [{ fromPort: 3306 - ipProtocol: tcp + ipProtocol: "tcp" ipRanges: [{ cidrIp: "\(parameter.vpcCidr)" }] diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue index 3accae9c5..7fbf22e3b 100644 --- a/platform/components/rds-cluster-mysql.cue +++ b/platform/components/rds-cluster-mysql.cue @@ -77,7 +77,7 @@ template: { region: "\(parameter.region)" ingress: [{ fromPort: 3306 - ipProtocol: tcp + ipProtocol: "tcp" ipRanges: [{ cidrIp: "\(parameter.vpcCidr)" }] diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 09c4c066e..42791551b 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -55,20 +55,22 @@ sed -e "s/INGRESS_DNS/${DNS_HOSTNAME}/g" ${REPO_ROOT}/platform/infra/terraform/m export GITHUB_URL=$(yq '.repo_url' ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/config.yaml) export GITHUB_BRANCH=$(yq '.repo_branch' ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/config.yaml) -VPC_ID=$(terraform output -raw eks_cluster_vpc_id) -SUBNET_IDS=$(terraform output -json database_subnet_ids | jq -r '.[]') +cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ -# Create array from subnet IDs -readarray -t SUBNET_ARRAY <<< "$SUBNET_IDS" +VPC_ID=$(terraform output -raw eks_cluster_vpc_id) +SUBNET_ID_1=$(terraform output -json database_subnet_ids | jq -r '.[0]') +SUBNET_ID_2=$(terraform output -json database_subnet_ids | jq -r '.[1]') +SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') # Generate the ConfigMap cat > ${REPO_ROOT}/packages/devlake/dev/infrastructure-values.env << EOF vpc_id=$VPC_ID -subnet_id_1=${SUBNET_ARRAY[0]} -subnet_id_2=${SUBNET_ARRAY[1]} -subnet_id_3=${SUBNET_ARRAY[2]} +subnet_id_1=$SUBNET_ID_1 +subnet_id_2=$SUBNET_ID_2 +subnet_id_3=$SUBNET_ID_3 EOF +cd ${REPO_ROOT}/platform/infra/terraform # Deploy the apps on IDP Builder and ArgoCD ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh From 237a698a7bfc96b195bde9d3b98cd08f3f0b3552 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 16 Apr 2025 14:05:22 -0700 Subject: [PATCH 29/63] change to crossplane xrd with environment config --- .../components/rds-cluster-mysql.yaml | 83 ++------ packages/devlake/dev/kustomization.yaml | 48 +---- packages/devlake/dev/mysql-init-job.yaml | 12 +- packages/devlake/dev/rds-mysql.yaml | 8 - platform/components/rds-cluster-mysql.cue | 104 ++-------- .../compositions/rds/mysql-aurora.yaml | 182 ++++++++++++++++++ .../terraform/scripts/crossplane/.gitignore | 2 + .../crossplane/cp-mgmt-env-config.yaml | 13 ++ .../terraform/scripts/devlake/devlake-init.sh | 34 ++-- .../terraform/scripts/devlake/project-init.sh | 10 +- .../infra/terraform/setup-environments.sh | 27 ++- 11 files changed, 288 insertions(+), 235 deletions(-) create mode 100644 platform/crossplane/compositions/rds/mysql-aurora.yaml create mode 100644 platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore create mode 100644 platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml index f5059e69f..df7a0c3fe 100644 --- a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -10,80 +10,27 @@ spec: cue: template: | output: { - apiVersion: "rds.aws.upbound.io/v1beta1" - kind: "Cluster" - metadata: name: "\(context.name)-cluster" + apiVersion: "awsblueprints.io/v1alpha1" + kind: "RelationalDatabase" + metadata: { + name: context.name + namespace: context.namespace + } spec: { - forProvider: { - autoGeneratePassword: true - engine: "aurora-mysql" - masterPasswordSecretRef: { - key: "password" - name: "\(context.name)-cluster-password" - namespace: "vela-system" + resourceConfig: { + name: context.name + deletionPolicy: "Delete" + tags: { + Name: context.name + "crossplane-managed": "true" } - masterUsername: "awsrdsadmin" - region: "\(parameter.region)" - dbSubnetGroupNameRef: name: "\(context.name)-subnetgroup" - skipFinalSnapshot: true } + databaseName: "\(context.name)db" writeConnectionSecretToRef: { - name: "\(context.name)-cluster-connection" - namespace: "vela-system" - } - } - } - outputs: { - "\(context.name)-subnetgroup": { - apiVersion: "rds.aws.upbound.io/v1beta1" - kind: "SubnetGroup" - metadata: name: "\(context.name)-subnetgroup" - spec: forProvider: { - region: "\(parameter.region)" - subnetIds: [ for subnetId in parameter.subnetIds {"\(subnetId)"}] + name: "\(context.name)-connection" + namespace: context.namespace } } - - "\(context.name)-instance": { - apiVersion: "rds.aws.upbound.io/v1beta1" - kind: "ClusterInstance" - metadata: name: "\(context.name)-clusterinstance" - spec: forProvider: { - region: "\(parameter.region)" - clusterIdentifierRef: name: "\(context.name)-cluster" - engine: "aurora-mysql" - instanceClass: "db.r5.large" - vpcSecurityGroupIDRefs: [{ - name: "\(context.name)-securitygroup" - }] - } - } - "\(context.name)-securitygroup": { - apiVersion: "ec2.aws.crossplane.io/v1beta1" - kind: "SecurityGroup" - metadata: name: "\(context.name)-securitygroup" - spec: forProvider: { - description: "SG for traffic to RDS from VPC" - groupName: "\(context.name)-securitygroup" - region: "\(parameter.region)" - ingress: [{ - fromPort: 3306 - ipProtocol: "tcp" - ipRanges: [{ - cidrIp: "\(parameter.vpcCidr)" - }] - toPort: 3306 - }] - vpcId: "\(parameter.vpcId)" - } - } - - } - parameter: { - region: string - subnetIds: [...string] - vpcId: string - vpcCidr: string } workload: definition: diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 22ad11a0e..d02d4e7df 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -1,42 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - - external-secrets.yaml - - mysql-init-job.yaml - - rds-mysql.yaml - -patchesStrategicMerge: - - patches/devlake-patch-sa.yaml - -configMapGenerator: -- name: vpc-config - envs: - - infrastructure-values.env - -vars: -- name: SUBNET_ID_1 - objref: - kind: ConfigMap - name: vpc-config - apiVersion: v1 - fieldref: - fieldpath: data.subnet_id_1 -- name: SUBNET_ID_2 - objref: - kind: ConfigMap - name: vpc-config - apiVersion: v1 - fieldref: - fieldpath: data.subnet_id_2 -- name: SUBNET_ID_3 - objref: - kind: ConfigMap - name: vpc-config - apiVersion: v1 - fieldref: - fieldpath: data.subnet_id_3 -- name: VPC_ID - objref: - kind: ConfigMap - name: vpc-config - apiVersion: v1 - fieldref: - fieldpath: data.vpc_id +- external-secrets.yaml +- mysql-init-job.yaml +- rds-mysql.yaml +patches: +- path: patches/devlake-patch-sa.yaml diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml index e71bc0f5d..b599fe916 100644 --- a/packages/devlake/dev/mysql-init-job.yaml +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -30,9 +30,9 @@ spec: - /bin/bash - -c - | - endpoint=$(kubectl get secret devlake-mysql-db-cluster-connection -n vela-system -o jsonpath='{.data.endpoint}' | base64 --decode) + endpoint=$(kubectl get secret devlake-mysql-db-connection -n devlake -o jsonpath='{.data.endpoint}' | base64 --decode) kubectl create service externalname devlake-mysql-service --external-name=$endpoint -n devlake --dry-run=client -o yaml | kubectl apply -f - - password=$(kubectl get secret devlake-mysql-db-cluster-password -n vela-system -o jsonpath='{.data.password}' | base64 --decode) + password=$(kubectl get secret devlake-mysql-db-connection -n devlake -o jsonpath='{.data.password}' | base64 --decode) kubectl create secret generic devlake-mysql-auth \ --from-literal=MYSQL_USER=merico \ --from-literal=MYSQL_PASSWORD=merico \ @@ -46,24 +46,24 @@ spec: - /bin/bash - -c - | - until mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do + until mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do echo "Waiting for MySQL to be ready..." sleep 2 done - mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " CREATE DATABASE IF NOT EXISTS lake; " echo "Creating devlake user..." - mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " CREATE USER IF NOT EXISTS 'merico' IDENTIFIED BY 'merico'; GRANT ALL PRIVILEGES ON lake.* TO 'merico'; FLUSH PRIVILEGES; " echo "Creating Grafana user..." - mysql -h ${DB_URL} -P 3306 -u awsrdsadmin -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password'; GRANT SELECT ON lake.* TO 'grafanaReader'; FLUSH PRIVILEGES; diff --git a/packages/devlake/dev/rds-mysql.yaml b/packages/devlake/dev/rds-mysql.yaml index ec05493f6..143d316c2 100644 --- a/packages/devlake/dev/rds-mysql.yaml +++ b/packages/devlake/dev/rds-mysql.yaml @@ -7,14 +7,6 @@ spec: components: - name: devlake-mysql-db type: rds-cluster-mysql - properties: - region: us-west-2 - subnetIds: - - $(SUBNET_ID_1) - - $(SUBNET_ID_2) - - $(SUBNET_ID_3) - vpcId: $(VPC_ID) - vpcCidr: 10.0.0.0/16 traits: - type: component-iam-policy properties: diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue index 7fbf22e3b..93c092d55 100644 --- a/platform/components/rds-cluster-mysql.cue +++ b/platform/components/rds-cluster-mysql.cue @@ -11,89 +11,27 @@ } template: { - output: { - apiVersion: "rds.aws.upbound.io/v1beta1" - kind: "Cluster" - metadata: { - name: "\(context.name)-cluster" - } - spec: { - forProvider: { - autoGeneratePassword: true - engine: "aurora-mysql" - masterPasswordSecretRef: { - key: "password" - name: "\(context.name)-cluster-password" - namespace: "vela-system" - } - masterUsername: "awsrdsadmin" - region: "\(parameter.region)" - dbSubnetGroupNameRef: name: "\(context.name)-subnetgroup" - skipFinalSnapshot: true - } - writeConnectionSecretToRef: { - name: "\(context.name)-cluster-connection" - namespace: "vela-system" - } - } - } - outputs: { - "\(context.name)-subnetgroup": { - apiVersion: "rds.aws.upbound.io/v1beta1" - kind: "SubnetGroup" - metadata: name: "\(context.name)-subnetgroup" - spec: { - forProvider: { - region: "\(parameter.region)" - subnetIds: [ for subnetId in parameter.subnetIds {"\(subnetId)"}] - } - } - } - - "\(context.name)-instance": { - apiVersion: "rds.aws.upbound.io/v1beta1" - kind: "ClusterInstance" - metadata: name: "\(context.name)-clusterinstance" - spec: { - forProvider: { - region: "\(parameter.region)" - clusterIdentifierRef: name: "\(context.name)-cluster" - engine: "aurora-mysql" - instanceClass: "db.r5.large" - vpcSecurityGroupIDRefs: [{ - name: "\(context.name)-securitygroup" - }] - } - } - } - "\(context.name)-securitygroup": { - apiVersion: "ec2.aws.crossplane.io/v1beta1" - kind: "SecurityGroup" - metadata: name: "\(context.name)-securitygroup" - spec: { - forProvider: { - description: "SG for traffic to RDS from VPC" - groupName: "\(context.name)-securitygroup" - region: "\(parameter.region)" - ingress: [{ - fromPort: 3306 - ipProtocol: "tcp" - ipRanges: [{ - cidrIp: "\(parameter.vpcCidr)" - }] - toPort: 3306 - }] - vpcId: "\(parameter.vpcId)" + output: { + apiVersion: "awsblueprints.io/v1alpha1" + kind: "RelationalDatabase" + metadata: { + name: context.name + namespace: context.namespace + } + spec: { + resourceConfig: { + name: context.name + deletionPolicy: "Delete" + tags: { + Name: context.name + "crossplane-managed": "true" + } + } + databaseName: "\(context.name)db" + writeConnectionSecretToRef: { + name: "\(context.name)-connection" + namespace: context.namespace + } } - } } - - - } - parameter: { - region: string - subnetIds: [...string] - vpcId: string - vpcCidr: string - } } diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml new file mode 100644 index 000000000..f46eaa82c --- /dev/null +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -0,0 +1,182 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: rds-aurora-mysql.awsblueprints.io + labels: + awsblueprints.io/provider: aws + awsblueprints.io/environment: staging + awsblueprints.io/createDBSubnetGroup: "true" +spec: + environment: + environmentConfigs: + - type: Reference + ref: + name: mgmt-environment + writeConnectionSecretsToNamespace: crossplane-system + compositeTypeRef: + apiVersion: awsblueprints.io/v1alpha1 + kind: XRelationalDatabase + patchSets: + - name: common-fields + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.resourceConfig.deletionPolicy + toFieldPath: spec.deletionPolicy + - type: FromEnvironmentFieldPath + fromFieldPath: locations.us + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.resourceConfig.tags + toFieldPath: spec.forProvider.tags + policy: + mergeOptions: + appendSlice: true + - type: FromCompositeFieldPath + fromFieldPath: spec.writeConnectionSecretToRef.namespace + toFieldPath: spec.writeConnectionSecretToRef.namespace + - type: FromCompositeFieldPath + fromFieldPath: spec.resourceConfig.name + toFieldPath: metadata.annotations[crossplane.io/external-name] + resources: + - base: + apiVersion: database.aws.crossplane.io/v1beta1 + kind: DBSubnetGroup + spec: + forProvider: + description: "rds-mysql" + patches: + - type: PatchSet + patchSetName: common-fields + - type: FromEnvironmentFieldPath + fromFieldPath: "dbSubnetIds" + toFieldPath: spec.forProvider.subnetIds + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-dbsubnet" + - base: + apiVersion: ec2.aws.crossplane.io/v1beta1 + kind: SecurityGroup + spec: + forProvider: + description: "rds-mysql-sg" + ingress: + - ipProtocol: "tcp" + fromPort: 3306 + toPort: 3306 + ipRanges: + - cidrIp: "10.0.0.0/8" + patches: + - type: PatchSet + patchSetName: common-fields + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.securityGroupID + toFieldPath: status.securityGroupId + - type: FromCompositeFieldPath + fromFieldPath: spec.IngressRules + toFieldPath: spec.forProvider.ingress + policy: + mergeOptions: + appendSlice: true + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.forProvider.groupName" + transforms: + - type: string + string: + fmt: "rds-mysql-sg-%s" + - type: FromEnvironmentFieldPath + fromFieldPath: "vpcId" + toFieldPath: "spec.forProvider.vpcId" + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-db-sg" + - base: + apiVersion: rds.aws.crossplane.io/v1alpha1 + kind: DBCluster + spec: + forProvider: + backupRetentionPeriod: 7 + copyTagsToSnapshot: true + dbSubnetGroupNameSelector: + matchControllerRef: true + enableHTTPEndpoint: false + engine: aurora-mysql + engineVersion: "8.0.mysql_aurora.3.05.2" + finalDBSnapshotIdentifier: "to-be-patched" + masterUserPasswordSecretRef: + key: password + name: mysql-root-user-password + namespace: crossplane-system + masterUsername: root + preferredBackupWindow: 02:00-03:00 + preferredMaintenanceWindow: sun:04:00-sun:05:00 + skipFinalSnapshot: false + storageEncrypted: true + vpcSecurityGroupIDs: [] + # This is a workaround for SGs not getting updated properly. https://github.com/crossplane/provider-aws/issues/1153 + # https://github.com/crossplane/crossplane-runtime/issues/250 + vpcSecurityGroupIDSelector: + matchControllerRef: true + patches: + - type: PatchSet + patchSetName: common-fields + - fromFieldPath: spec.writeConnectionSecretToRef.namespace + toFieldPath: spec.writeConnectionSecretToRef.namespace + - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.forProvider.masterUserPasswordSecretRef.namespace + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-mysql" + - fromFieldPath: "metadata.name" + toFieldPath: "spec.forProvider.finalDBSnapshotIdentifier" + transforms: + - type: string + string: + fmt: "%s-final-snapshot" + - fromFieldPath: "spec.engineVersion" + toFieldPath: "spec.forProvider.engineVersion" + - fromFieldPath: "spec.databaseName" + toFieldPath: "spec.forProvider.databaseName" + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.dbClusterIdentifier + toFieldPath: status.clusterId + connectionDetails: + - fromConnectionSecretKey: username + - fromConnectionSecretKey: password + - fromConnectionSecretKey: endpoint + - fromConnectionSecretKey: port + - base: + apiVersion: rds.aws.crossplane.io/v1alpha1 + kind: DBInstance + spec: + forProvider: + autogeneratePassword: true + dbClusterIdentifier: "to-be-patched" + dbSubnetGroupNameSelector: + matchControllerRef: true + dbInstanceClass: db.r5.xlarge + engine: aurora-mysql + patches: + - type: PatchSet + patchSetName: common-fields + - fromFieldPath: spec.writeConnectionSecretToRef.namespace + toFieldPath: spec.writeConnectionSecretToRef.namespace + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-cluster-mysql" + - fromFieldPath: "status.clusterId" + toFieldPath: "spec.forProvider.dbClusterIdentifier" diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore new file mode 100644 index 000000000..eed59f21e --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore @@ -0,0 +1,2 @@ +*.yml + diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml new file mode 100644 index 000000000..62939a2fd --- /dev/null +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.crossplane.io/v1alpha1 +kind: EnvironmentConfig +metadata: + name: mgmt-environment +data: + locations: + us: us-west-2 + dbSubnetIds: + - $SUBNET_ID_1 + - $SUBNET_ID_2 + - $SUBNET_ID_3 + vpcId: $VPC_ID + diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh index c080c7e02..d6b88fc3a 100755 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh @@ -17,21 +17,25 @@ done OS=$(uname -s) -STRINGS=("dotnet" "go" "java" "nextjs" "") +STRINGS=("dotnet" "go" "java" "nextjs") for str in "${STRINGS[@]}"; do -./project-init.sh $str - process_args "$MAC_SCRIPT" "$str" + key_conn_bp=$(./project-init.sh $str) + key=$(echo key_conn_bp | cut -d'|' -f1) + conn=$(echo key_conn_bp | cut -d'|' -f2) + bp=$(echo key_conn_bp | cut -d'|' -f3) + case "$OS" in + "Darwin") + ./generate-data-mac.sh http://localhost:9090 $key $conn + ;; + "Linux") + ./generate-data-linux.sh http://localhost:9090 $key $conn + ;; + *) + echo "Unsupported operating system: $OS" + exit 1 + ;; + esac + curl -X POST localhost:9090/blueprints/$bp/trigger + done -case "$OS" in - "Darwin") - ./generate-data-mac.sh - ;; - "Linux") - ./generate-data-linux.sh - ;; - *) - echo "Unsupported operating system: $OS" - exit 1 - ;; -esac diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh index 23f43e481..499fb9c6e 100755 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/project-init.sh @@ -7,8 +7,7 @@ if [ $# -lt 1 ]; then fi -echo "Performing DB Migration" -curl -X GET localhost:9090/proceed-db-migration +resp=$(curl -X GET localhost:9090/proceed-db-migration) projectName=$1 @@ -32,15 +31,11 @@ projectCreateRequest=$( } EOF ) -echo $projectCreateRequest -echo "Creating Project: $projectName" projectCreateResponse=$(curl -X POST -H "Content-Type: application/json" localhost:9090/projects -d "$projectCreateRequest") -echo $projectCreateResponse blueprintID=$(echo $projectCreateResponse | jq -r '.blueprint.id') blueprintName=$(echo $projectCreateResponse | jq -r '.blueprint.name') # {"name":"modengg","description":"","createdAt":"2025-03-21T23:46:01.144Z","updatedAt":"2025-03-21T23:46:01.144Z","_raw_data_params":"","_raw_data_table":"","_raw_data_id":0,"_raw_data_remark":"","metrics":[{"pluginName":"dora","pluginOption":{},"enable":true},{"pluginName":"issue_trace","pluginOption":{},"enable":true}],"blueprint":{"name":"test-Blueprint","projectName":"test","mode":"NORMAL","plan":null,"enable":true,"cronConfig":"0 0 * * 1","isManual":false,"beforePlan":null,"afterPlan":null,"labels":[],"connections":[],"skipOnFail":false,"timeAfter":"2024-09-21T00:00:00Z","skipCollectors":false,"fullSync":false,"id":1,"createdAt":"2025-03-21T23:46:01.148Z","updatedAt":"2025-03-21T23:46:01.148Z"}} -echo "Creating Webhook" webhookCreateRequest=$( cat < ${REPO_ROOT}/packages/devlake/dev/infrastructure-values.env << EOF -vpc_id=$VPC_ID -subnet_id_1=$SUBNET_ID_1 -subnet_id_2=$SUBNET_ID_2 -subnet_id_3=$SUBNET_ID_3 -EOF +# Setup CP Env Config for MGMT Cluster +sed -e "s#\$SUBNET_ID_1#${SUBNET_ID_1}#g" -e "s#\$SUBNET_ID_2#${SUBNET_ID_2}#g" -e "s#\$SUBNET_ID_3#${SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/scripts/crossplane/cp-mgmt-env-config.yml +kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/scripts/crossplane/cp-mgmt-env-config.yml cd ${REPO_ROOT}/platform/infra/terraform @@ -220,6 +216,23 @@ terraform -chdir=dev apply "devplan" & export DEV_EKS_PROCESS=$! +# # Initialize backend for DB PROD cluster +# terraform -chdir=prod/db init -reconfigure -backend-config="key=prod/db/db-ec2-cluster.tfstate" \ +# -backend-config="bucket=$TF_VAR_state_s3_bucket" \ +# -backend-config="region=$TF_VAR_aws_region" \ +# -backend-config="dynamodb_table=$TF_VAR_state_ddb_lock_table" +# +# # Apply the infrastructure changes to deploy DB PROD cluster +# terraform -chdir=prod/db plan -var aws_region="${TF_VAR_aws_region}" \ +# -var vpc_id="${TF_eks_cluster_vpc_id}" \ +# -var vpc_private_subnets="${TF_eks_cluster_private_subnets}" \ +# -var availability_zones="${TF_eks_cluster_private_az}" \ +# -var vpc_cidr="${TF_eks_cluster_vpc_cidr}" \ +# -var key_name="ws-default-keypair" -var region="${TF_VAR_aws_region}" -out=proddbplan +# +# terraform -chdir=prod/db apply "proddbplan" & +# +# export prod_DB_PROCESS=$! # Initialize backend for PROD cluster terraform -chdir=prod init -reconfigure -backend-config="key=prod/eks-accelerator-vpc.tfstate" \ From 60cd03a97d3273485160191dc25b982b47d97ef7 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 16 Apr 2025 14:15:33 -0700 Subject: [PATCH 30/63] get rid of patch fix mysql fix mysql composition fix msql comp 2 add ec2 provider and fix mysql fix more fixes fixes another try another try2 get rid of tags for now fix instance fix sg ids get rid of external name password fix fixes to crossplane compositions and kubevela template --- .../components/rds-cluster-mysql.yaml | 25 ++-- .../base/provider-aws.yaml | 11 ++ packages/devlake/dev/kustomization.yaml | 4 +- packages/devlake/dev/mysql-init-job.yaml | 7 +- platform/components/rds-cluster-mysql.cue | 27 ++-- .../compositions/rds/mysql-aurora.yaml | 119 +++++++++--------- .../argocd-apps/crossplane-compositions.yaml | 2 + .../infra/terraform/setup-environments.sh | 22 +--- 8 files changed, 116 insertions(+), 101 deletions(-) diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml index df7a0c3fe..d186bd9b0 100644 --- a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -17,19 +17,28 @@ spec: namespace: context.namespace } spec: { + compositionSelector: matchLabels: { + "awsblueprints.io/provider": "aws" + "awsblueprints.io/environment": "prod" + "awsblueprints.io/createDBSubnetGroup": "true" + } resourceConfig: { name: context.name deletionPolicy: "Delete" - tags: { - Name: context.name - "crossplane-managed": "true" - } + tags: [ + { + key: "env" + value: "prod" + }, + { + key: "app" + value: "\(context.name)" + }, + ] + providerConfigName: "default" + region: "us-west-2" } databaseName: "\(context.name)db" - writeConnectionSecretToRef: { - name: "\(context.name)-connection" - namespace: context.namespace - } } } workload: diff --git a/packages/crossplane-provider/base/provider-aws.yaml b/packages/crossplane-provider/base/provider-aws.yaml index 678325ab1..f5da7d43e 100644 --- a/packages/crossplane-provider/base/provider-aws.yaml +++ b/packages/crossplane-provider/base/provider-aws.yaml @@ -83,3 +83,14 @@ spec: package: xpkg.upbound.io/upbound/provider-aws-rds:v1.13.1 runtimeConfigRef: name: aws-irsa-drc +--- +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-ec2 + annotations: + argocd.argoproj.io/sync-wave: "10" +spec: + package: xpkg.upbound.io/upbound/provider-aws-ec2:v1.13.1 + runtimeConfigRef: + name: aws-irsa-drc diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index d02d4e7df..58be0507c 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -4,5 +4,5 @@ resources: - external-secrets.yaml - mysql-init-job.yaml - rds-mysql.yaml -patches: -- path: patches/devlake-patch-sa.yaml +# patches: +# - path: patches/devlake-patch-sa.yaml diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml index b599fe916..b9ab104f0 100644 --- a/packages/devlake/dev/mysql-init-job.yaml +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -30,10 +30,11 @@ spec: - /bin/bash - -c - | - endpoint=$(kubectl get secret devlake-mysql-db-connection -n devlake -o jsonpath='{.data.endpoint}' | base64 --decode) + CONN_SECRET_NAME=$(kubectl get secrets -o custom-columns=NAME:.metadata.name | grep "^devlake-mysql.*cluster-mysql-connection$") + endpoint=$(kubectl get secret $CONN_SECRET_NAME -n crossplane-system -o jsonpath='{.data.endpoint}' | base64 --decode) kubectl create service externalname devlake-mysql-service --external-name=$endpoint -n devlake --dry-run=client -o yaml | kubectl apply -f - - password=$(kubectl get secret devlake-mysql-db-connection -n devlake -o jsonpath='{.data.password}' | base64 --decode) - kubectl create secret generic devlake-mysql-auth \ + password=$(kubectl get secret $CONN_SECRET_NAME -n crossplane-system -o jsonpath='{.data.attribute\.master_password}' | base64 --decode) + kubectl create secret generic devlake-mysql-auth -n devlake \ --from-literal=MYSQL_USER=merico \ --from-literal=MYSQL_PASSWORD=merico \ --from-literal=MYSQL_DATABASE=lake \ diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue index 93c092d55..73727fd4e 100644 --- a/platform/components/rds-cluster-mysql.cue +++ b/platform/components/rds-cluster-mysql.cue @@ -19,19 +19,30 @@ template: { namespace: context.namespace } spec: { + compositionSelector: { + matchLabels: { + "awsblueprints.io/provider": "aws" + "awsblueprints.io/environment": "prod" + "awsblueprints.io/createDBSubnetGroup": "true" + } + } resourceConfig: { name: context.name deletionPolicy: "Delete" - tags: { - Name: context.name - "crossplane-managed": "true" - } + tags: [ + { + key: "env" + value: "prod" + }, + { + key: "app" + value: "\(context.name)" + }, + ] + providerConfigName: "default" + region: "us-west-2" } databaseName: "\(context.name)db" - writeConnectionSecretToRef: { - name: "\(context.name)-connection" - namespace: context.namespace - } } } } diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml index f46eaa82c..a089d0a4e 100644 --- a/platform/crossplane/compositions/rds/mysql-aurora.yaml +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -7,7 +7,7 @@ metadata: name: rds-aurora-mysql.awsblueprints.io labels: awsblueprints.io/provider: aws - awsblueprints.io/environment: staging + awsblueprints.io/environment: prod awsblueprints.io/createDBSubnetGroup: "true" spec: environment: @@ -28,55 +28,45 @@ spec: - type: FromEnvironmentFieldPath fromFieldPath: locations.us toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: spec.resourceConfig.tags - toFieldPath: spec.forProvider.tags - policy: - mergeOptions: - appendSlice: true - type: FromCompositeFieldPath fromFieldPath: spec.writeConnectionSecretToRef.namespace toFieldPath: spec.writeConnectionSecretToRef.namespace - - type: FromCompositeFieldPath - fromFieldPath: spec.resourceConfig.name - toFieldPath: metadata.annotations[crossplane.io/external-name] + - name: connection-secret + patches: + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-mysql" + - fromFieldPath: "spec.writeConnectionSecretToRef.namespace" + toFieldPath: "spec.writeConnectionSecretToRef.namespace" resources: - base: - apiVersion: database.aws.crossplane.io/v1beta1 - kind: DBSubnetGroup + apiVersion: rds.aws.upbound.io/v1beta1 + kind: SubnetGroup spec: forProvider: description: "rds-mysql" patches: - type: PatchSet patchSetName: common-fields + - type: PatchSet + patchSetName: connection-secret - type: FromEnvironmentFieldPath fromFieldPath: "dbSubnetIds" toFieldPath: spec.forProvider.subnetIds - - fromFieldPath: "metadata.uid" - toFieldPath: "spec.writeConnectionSecretToRef.name" - transforms: - - type: string - string: - fmt: "%s-dbsubnet" - base: - apiVersion: ec2.aws.crossplane.io/v1beta1 + apiVersion: ec2.aws.upbound.io/v1beta1 kind: SecurityGroup spec: forProvider: description: "rds-mysql-sg" - ingress: - - ipProtocol: "tcp" - fromPort: 3306 - toPort: 3306 - ipRanges: - - cidrIp: "10.0.0.0/8" patches: - type: PatchSet patchSetName: common-fields - - type: ToCompositeFieldPath - fromFieldPath: status.atProvider.securityGroupID - toFieldPath: status.securityGroupId + - type: PatchSet + patchSetName: connection-secret - type: FromCompositeFieldPath fromFieldPath: spec.IngressRules toFieldPath: spec.forProvider.ingress @@ -84,7 +74,7 @@ spec: mergeOptions: appendSlice: true - fromFieldPath: "metadata.uid" - toFieldPath: "spec.forProvider.groupName" + toFieldPath: "spec.forProvider.name" transforms: - type: string string: @@ -92,38 +82,46 @@ spec: - type: FromEnvironmentFieldPath fromFieldPath: "vpcId" toFieldPath: "spec.forProvider.vpcId" - - fromFieldPath: "metadata.uid" - toFieldPath: "spec.writeConnectionSecretToRef.name" - transforms: - - type: string - string: - fmt: "%s-db-sg" - base: - apiVersion: rds.aws.crossplane.io/v1alpha1 - kind: DBCluster + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroupIngressRule spec: forProvider: + fromPort: 3306 + toPort: 3306 + ipProtocol: "tcp" + cidrIpv4: "10.0.0.0/8" + securityGroupIdSelector: + matchControllerRef: true + patches: + - type: PatchSet + patchSetName: common-fields + - type: PatchSet + patchSetName: connection-secret + + - base: + apiVersion: rds.aws.upbound.io/v1beta2 + kind: Cluster + spec: + forProvider: + autoGeneratePassword: true backupRetentionPeriod: 7 copyTagsToSnapshot: true - dbSubnetGroupNameSelector: - matchControllerRef: true - enableHTTPEndpoint: false engine: aurora-mysql engineVersion: "8.0.mysql_aurora.3.05.2" finalDBSnapshotIdentifier: "to-be-patched" - masterUserPasswordSecretRef: + masterPasswordSecretRef: key: password name: mysql-root-user-password namespace: crossplane-system masterUsername: root preferredBackupWindow: 02:00-03:00 preferredMaintenanceWindow: sun:04:00-sun:05:00 - skipFinalSnapshot: false + skipFinalSnapshot: true storageEncrypted: true - vpcSecurityGroupIDs: [] - # This is a workaround for SGs not getting updated properly. https://github.com/crossplane/provider-aws/issues/1153 - # https://github.com/crossplane/crossplane-runtime/issues/250 - vpcSecurityGroupIDSelector: + dbSubnetGroupNameSelector: + matchControllerRef: true + vpcSecurityGroupIdSelector: matchControllerRef: true patches: - type: PatchSet @@ -132,12 +130,18 @@ spec: toFieldPath: spec.writeConnectionSecretToRef.namespace - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] toFieldPath: spec.forProvider.masterUserPasswordSecretRef.namespace - - fromFieldPath: "metadata.uid" + - fromFieldPath: "metadata.name" + toFieldPath: "spec.masterPasswordSecretRef.name" + transforms: + - type: string + string: + fmt: "%s-cluster-mysql-master-password" + - fromFieldPath: "metadata.name" toFieldPath: "spec.writeConnectionSecretToRef.name" transforms: - type: string string: - fmt: "%s-mysql" + fmt: "%s-cluster-mysql-connection" - fromFieldPath: "metadata.name" toFieldPath: "spec.forProvider.finalDBSnapshotIdentifier" transforms: @@ -148,35 +152,30 @@ spec: toFieldPath: "spec.forProvider.engineVersion" - fromFieldPath: "spec.databaseName" toFieldPath: "spec.forProvider.databaseName" - - type: ToCompositeFieldPath - fromFieldPath: status.atProvider.dbClusterIdentifier - toFieldPath: status.clusterId connectionDetails: - fromConnectionSecretKey: username - fromConnectionSecretKey: password - fromConnectionSecretKey: endpoint - fromConnectionSecretKey: port - base: - apiVersion: rds.aws.crossplane.io/v1alpha1 - kind: DBInstance + apiVersion: rds.aws.upbound.io/v1beta1 + kind: ClusterInstance spec: forProvider: - autogeneratePassword: true - dbClusterIdentifier: "to-be-patched" + clusterIdentifierSelector: + matchControllerRef: true dbSubnetGroupNameSelector: matchControllerRef: true - dbInstanceClass: db.r5.xlarge + instanceClass: db.r5.xlarge engine: aurora-mysql patches: - type: PatchSet patchSetName: common-fields - fromFieldPath: spec.writeConnectionSecretToRef.namespace toFieldPath: spec.writeConnectionSecretToRef.namespace - - fromFieldPath: "metadata.uid" + - fromFieldPath: "metadata.name" toFieldPath: "spec.writeConnectionSecretToRef.name" transforms: - type: string string: - fmt: "%s-cluster-mysql" - - fromFieldPath: "status.clusterId" - toFieldPath: "spec.forProvider.dbClusterIdentifier" + fmt: "%s-instance-mysql" diff --git a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml index 5afd3f73a..8f0823f57 100644 --- a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml +++ b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml @@ -13,6 +13,8 @@ spec: repoURL: ${GITHUB_URL} targetRevision: ${GITHUB_BRANCH} path: "platform/crossplane/compositions" + directory: + recurse: true destination: server: "https://kubernetes.default.svc" namespace: crossplane-system diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index e6696bbbb..8787ab8c1 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -63,8 +63,8 @@ SUBNET_ID_2=$(terraform output -json database_subnet_ids | jq -r '.[1]') SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') # Setup CP Env Config for MGMT Cluster -sed -e "s#\$SUBNET_ID_1#${SUBNET_ID_1}#g" -e "s#\$SUBNET_ID_2#${SUBNET_ID_2}#g" -e "s#\$SUBNET_ID_3#${SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/scripts/crossplane/cp-mgmt-env-config.yml -kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/scripts/crossplane/cp-mgmt-env-config.yml +sed -e "s#\$SUBNET_ID_1#${SUBNET_ID_1}#g" -e "s#\$SUBNET_ID_2#${SUBNET_ID_2}#g" -e "s#\$SUBNET_ID_3#${SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml +kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml cd ${REPO_ROOT}/platform/infra/terraform @@ -216,24 +216,6 @@ terraform -chdir=dev apply "devplan" & export DEV_EKS_PROCESS=$! -# # Initialize backend for DB PROD cluster -# terraform -chdir=prod/db init -reconfigure -backend-config="key=prod/db/db-ec2-cluster.tfstate" \ -# -backend-config="bucket=$TF_VAR_state_s3_bucket" \ -# -backend-config="region=$TF_VAR_aws_region" \ -# -backend-config="dynamodb_table=$TF_VAR_state_ddb_lock_table" -# -# # Apply the infrastructure changes to deploy DB PROD cluster -# terraform -chdir=prod/db plan -var aws_region="${TF_VAR_aws_region}" \ -# -var vpc_id="${TF_eks_cluster_vpc_id}" \ -# -var vpc_private_subnets="${TF_eks_cluster_private_subnets}" \ -# -var availability_zones="${TF_eks_cluster_private_az}" \ -# -var vpc_cidr="${TF_eks_cluster_vpc_cidr}" \ -# -var key_name="ws-default-keypair" -var region="${TF_VAR_aws_region}" -out=proddbplan -# -# terraform -chdir=prod/db apply "proddbplan" & -# -# export prod_DB_PROCESS=$! - # Initialize backend for PROD cluster terraform -chdir=prod init -reconfigure -backend-config="key=prod/eks-accelerator-vpc.tfstate" \ -backend-config="bucket=$TF_VAR_state_s3_bucket" \ From e60a4bee5b29bb7e9adbcd1d97b6a123ff367fa4 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 14:27:57 -0700 Subject: [PATCH 31/63] add more subnets to env config --- .../scripts/crossplane/cp-mgmt-env-config.yaml | 10 +++++++--- platform/infra/terraform/setup-environments.sh | 13 +++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml index 62939a2fd..fa4ef26ba 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml @@ -5,9 +5,13 @@ metadata: data: locations: us: us-west-2 + privateSubnetIds: + - $PRIV_SUBNET_ID_1 + - $PRIV_SUBNET_ID_2 + - $PRIV_SUBNET_ID_3 dbSubnetIds: - - $SUBNET_ID_1 - - $SUBNET_ID_2 - - $SUBNET_ID_3 + - $DB_SUBNET_ID_1 + - $DB_SUBNET_ID_2 + - $DB_SUBNET_ID_3 vpcId: $VPC_ID diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 8787ab8c1..c92fa0a25 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -58,12 +58,17 @@ export GITHUB_BRANCH=$(yq '.repo_branch' ${REPO_ROOT}/platform/infra/terraform/m cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ VPC_ID=$(terraform output -raw eks_cluster_vpc_id) -SUBNET_ID_1=$(terraform output -json database_subnet_ids | jq -r '.[0]') -SUBNET_ID_2=$(terraform output -json database_subnet_ids | jq -r '.[1]') -SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') + +PRIV_SUBNET_ID_1=$(terraform output -json eks_cluster_private_subnets | jq -r '.[0]') +PRIV_SUBNET_ID_2=$(terraform output -json eks_cluster_private_subnets | jq -r '.[1]') +PRIV_SUBNET_ID_3=$(terraform output -json eks_cluster_private_subnets | jq -r '.[2]') + +DB_SUBNET_ID_1=$(terraform output -json database_subnet_ids | jq -r '.[0]') +DB_SUBNET_ID_2=$(terraform output -json database_subnet_ids | jq -r '.[1]') +DB_SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') # Setup CP Env Config for MGMT Cluster -sed -e "s#\$SUBNET_ID_1#${SUBNET_ID_1}#g" -e "s#\$SUBNET_ID_2#${SUBNET_ID_2}#g" -e "s#\$SUBNET_ID_3#${SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml +sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$DB_SUBNET_ID_1#${DB_SUBNET_ID_1}#g" -e "s#\$DB_SUBNET_ID_2#${DB_SUBNET_ID_2}#g" -e "s#\$DB_SUBNET_ID_3#${DB_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml cd ${REPO_ROOT}/platform/infra/terraform From d3dca0ff4f265e965b34f572fb461348c0f9db92 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 16:03:38 -0700 Subject: [PATCH 32/63] fix dbname --- .../addons/kubevela/templates/components/rds-cluster-mysql.yaml | 2 +- platform/components/rds-cluster-mysql.cue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml index d186bd9b0..dc930318e 100644 --- a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -38,7 +38,7 @@ spec: providerConfigName: "default" region: "us-west-2" } - databaseName: "\(context.name)db" + databaseName: "initdb" } } workload: diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue index 73727fd4e..ad1dca5e4 100644 --- a/platform/components/rds-cluster-mysql.cue +++ b/platform/components/rds-cluster-mysql.cue @@ -42,7 +42,7 @@ template: { providerConfigName: "default" region: "us-west-2" } - databaseName: "\(context.name)db" + databaseName: "initdb" } } } From c40eb30e8413f20771c98541cb8454c34606e5c6 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 16:14:02 -0700 Subject: [PATCH 33/63] get rid of connection secrets for resources that don't need --- .../compositions/rds/mysql-aurora.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml index a089d0a4e..a23e4e426 100644 --- a/platform/crossplane/compositions/rds/mysql-aurora.yaml +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -28,9 +28,6 @@ spec: - type: FromEnvironmentFieldPath fromFieldPath: locations.us toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: spec.writeConnectionSecretToRef.namespace - toFieldPath: spec.writeConnectionSecretToRef.namespace - name: connection-secret patches: - fromFieldPath: "metadata.uid" @@ -51,8 +48,6 @@ spec: patches: - type: PatchSet patchSetName: common-fields - - type: PatchSet - patchSetName: connection-secret - type: FromEnvironmentFieldPath fromFieldPath: "dbSubnetIds" toFieldPath: spec.forProvider.subnetIds @@ -65,8 +60,6 @@ spec: patches: - type: PatchSet patchSetName: common-fields - - type: PatchSet - patchSetName: connection-secret - type: FromCompositeFieldPath fromFieldPath: spec.IngressRules toFieldPath: spec.forProvider.ingress @@ -96,8 +89,6 @@ spec: patches: - type: PatchSet patchSetName: common-fields - - type: PatchSet - patchSetName: connection-secret - base: apiVersion: rds.aws.upbound.io/v1beta2 @@ -171,11 +162,3 @@ spec: patches: - type: PatchSet patchSetName: common-fields - - fromFieldPath: spec.writeConnectionSecretToRef.namespace - toFieldPath: spec.writeConnectionSecretToRef.namespace - - fromFieldPath: "metadata.name" - toFieldPath: "spec.writeConnectionSecretToRef.name" - transforms: - - type: string - string: - fmt: "%s-instance-mysql" From db326ab23cebbc251686b0d07ec1b2d9f0650782 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 16:17:09 -0700 Subject: [PATCH 34/63] fix job more fixes fix fix job fix job 2 fix 3 another attempt testing testing --- packages/devlake/dev/mysql-init-job.yaml | 17 +++++++++++++---- .../compositions/rds/mysql-aurora.yaml | 5 +---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml index b9ab104f0..b01266162 100644 --- a/packages/devlake/dev/mysql-init-job.yaml +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -30,10 +30,19 @@ spec: - /bin/bash - -c - | - CONN_SECRET_NAME=$(kubectl get secrets -o custom-columns=NAME:.metadata.name | grep "^devlake-mysql.*cluster-mysql-connection$") - endpoint=$(kubectl get secret $CONN_SECRET_NAME -n crossplane-system -o jsonpath='{.data.endpoint}' | base64 --decode) - kubectl create service externalname devlake-mysql-service --external-name=$endpoint -n devlake --dry-run=client -o yaml | kubectl apply -f - - password=$(kubectl get secret $CONN_SECRET_NAME -n crossplane-system -o jsonpath='{.data.attribute\.master_password}' | base64 --decode) + while true; do + CONN_SECRET_NAME=$(kubectl get secrets -n crossplane-system -o custom-columns=NAME:.metadata.name | grep "^devlake-mysql.*cluster-mysql-connection$" || true) + if [ -n "$CONN_SECRET_NAME" ]; then + break + fi + echo "Waiting for secret to be available..." + sleep 5 + done + echo $CONN_SECRET_NAME + endpoint=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.endpoint}' | base64 --decode) + echo $endpoint + kubectl create service externalname devlake-mysql-service --external-name=${endpoint} -n devlake + password=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.attribute\.master_password}' | base64 --decode) kubectl create secret generic devlake-mysql-auth -n devlake \ --from-literal=MYSQL_USER=merico \ --from-literal=MYSQL_PASSWORD=merico \ diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml index a23e4e426..f233f1e50 100644 --- a/platform/crossplane/compositions/rds/mysql-aurora.yaml +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -103,7 +103,6 @@ spec: finalDBSnapshotIdentifier: "to-be-patched" masterPasswordSecretRef: key: password - name: mysql-root-user-password namespace: crossplane-system masterUsername: root preferredBackupWindow: 02:00-03:00 @@ -119,10 +118,8 @@ spec: patchSetName: common-fields - fromFieldPath: spec.writeConnectionSecretToRef.namespace toFieldPath: spec.writeConnectionSecretToRef.namespace - - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] - toFieldPath: spec.forProvider.masterUserPasswordSecretRef.namespace - fromFieldPath: "metadata.name" - toFieldPath: "spec.masterPasswordSecretRef.name" + toFieldPath: "spec.forProvider.masterPasswordSecretRef.name" transforms: - type: string string: From 0c991b98ec9ec157d872c6e5be639abc3f330532 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 16:49:55 -0700 Subject: [PATCH 35/63] fix dburl fix host --- packages/devlake/dev/mysql-init-job.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/devlake/dev/mysql-init-job.yaml b/packages/devlake/dev/mysql-init-job.yaml index b01266162..0a376f630 100644 --- a/packages/devlake/dev/mysql-init-job.yaml +++ b/packages/devlake/dev/mysql-init-job.yaml @@ -38,9 +38,7 @@ spec: echo "Waiting for secret to be available..." sleep 5 done - echo $CONN_SECRET_NAME endpoint=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.endpoint}' | base64 --decode) - echo $endpoint kubectl create service externalname devlake-mysql-service --external-name=${endpoint} -n devlake password=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.attribute\.master_password}' | base64 --decode) kubectl create secret generic devlake-mysql-auth -n devlake \ @@ -48,7 +46,7 @@ spec: --from-literal=MYSQL_PASSWORD=merico \ --from-literal=MYSQL_DATABASE=lake \ --from-literal=MYSQL_ROOT_PASSWORD=$password \ - --from-literal=DB_URL=mysql://merico:merico@devlake-mysql-service:3306/lake?charset=utf8mb4&parseTime=True + --from-literal=DB_URL="mysql://merico:merico@devlake-mysql-service:3306/lake?charset=utf8mb4&parseTime=True" containers: - name: create-grafana-user image: mysql:8 @@ -56,24 +54,24 @@ spec: - /bin/bash - -c - | - until mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do + until mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do echo "Waiting for MySQL to be ready..." sleep 2 done - mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " CREATE DATABASE IF NOT EXISTS lake; " echo "Creating devlake user..." - mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " CREATE USER IF NOT EXISTS 'merico' IDENTIFIED BY 'merico'; GRANT ALL PRIVILEGES ON lake.* TO 'merico'; FLUSH PRIVILEGES; " echo "Creating Grafana user..." - mysql -h ${DB_URL} -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " + mysql -h devlake-mysql-service -P 3306 -u root -p${MYSQL_ROOT_PASSWORD} -e " CREATE USER IF NOT EXISTS 'grafanaReader' IDENTIFIED BY 'grafana_password'; GRANT SELECT ON lake.* TO 'grafanaReader'; FLUSH PRIVILEGES; From 7b1c955bd0af164ce149bdf31ca52add9387dae1 Mon Sep 17 00:00:00 2001 From: Zach Jacobson <72418868+zjaco13@users.noreply.github.com> Date: Thu, 17 Apr 2025 17:22:58 -0700 Subject: [PATCH 36/63] Update catalog-info.yaml --- platform/backstage/templates/catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/backstage/templates/catalog-info.yaml b/platform/backstage/templates/catalog-info.yaml index d56158a2e..aab3994f8 100644 --- a/platform/backstage/templates/catalog-info.yaml +++ b/platform/backstage/templates/catalog-info.yaml @@ -35,4 +35,4 @@ metadata: description: Holds system information i.e, hostname, IP, OS, etc spec: owner: guest - hostname: modern-engg-d773ae6fd4aebb9d.elb.us-west-2.amazonaws.com + hostname: modern-engg-52a778290c6c8c1f.elb.us-west-2.amazonaws.com From 1857497983df0ffd08e950e164b0223a4f696eb6 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Thu, 17 Apr 2025 17:56:41 -0700 Subject: [PATCH 37/63] fix ordering --- platform/infra/terraform/setup-environments.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index c92fa0a25..b14e99764 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -55,6 +55,10 @@ sed -e "s/INGRESS_DNS/${DNS_HOSTNAME}/g" ${REPO_ROOT}/platform/infra/terraform/m export GITHUB_URL=$(yq '.repo_url' ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/config.yaml) export GITHUB_BRANCH=$(yq '.repo_branch' ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/config.yaml) +# Deploy the apps on IDP Builder and ArgoCD +${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh + + cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ VPC_ID=$(terraform output -raw eks_cluster_vpc_id) @@ -71,12 +75,6 @@ DB_SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$DB_SUBNET_ID_1#${DB_SUBNET_ID_1}#g" -e "s#\$DB_SUBNET_ID_2#${DB_SUBNET_ID_2}#g" -e "s#\$DB_SUBNET_ID_3#${DB_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml -cd ${REPO_ROOT}/platform/infra/terraform - -# Deploy the apps on IDP Builder and ArgoCD -${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh - -cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) echo "private subnets are : " $TF_eks_cluster_private_subnets From 9c360eb294599a4569a6028f51081c41ed83c3da Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Fri, 18 Apr 2025 11:28:28 -0700 Subject: [PATCH 38/63] fix provisioner --- .../dora-onboarding/provisioner/rbac.yaml | 3 +-- .../dora-onboarding/provisioner/wf-templates.yaml | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/platform/backstage/templates/dora-onboarding/provisioner/rbac.yaml b/platform/backstage/templates/dora-onboarding/provisioner/rbac.yaml index 4e9c1a9ea..b2e06e68a 100644 --- a/platform/backstage/templates/dora-onboarding/provisioner/rbac.yaml +++ b/platform/backstage/templates/dora-onboarding/provisioner/rbac.yaml @@ -5,10 +5,9 @@ metadata: namespace: ${{values.namespace}} --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: name: ${{values.appname}}-dora-provisioner-binding - namespace: ${{values.namespace}} subjects: - kind: ServiceAccount name: ${{values.appname}}-dora-provisioner diff --git a/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml b/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml index 4f5fabd0e..017e58e93 100644 --- a/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml +++ b/platform/backstage/templates/dora-onboarding/provisioner/wf-templates.yaml @@ -202,7 +202,8 @@ spec: - | apk add --no-cache yq set -ex - mkdir ./src + rm -rf src + mkdir src git clone https://"{{inputs.parameters.apprepo}}".git -c http.sslVerify=false ./src cd ./src git config --global user.email "noreply@aws.com" @@ -213,29 +214,31 @@ spec: yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.templates[0].steps += [[{ "name": "send-deployment-status", "templateRef": { - "name": "{{workflow.parameters.appname}}"-cicd-deployments-dora-processing-template", + "name": "{{workflow.parameters.appname}}-cicd-deployments-dora-processing-template", "template": "handle-rollout-check-and-process-deploy" }, "arguments": { "parameters": [ { "name": "body", - "value": 'PLACEHOLDER_OPENworkflow.parameters.bodyPLACEHOLDER_CLOSE' + "value": "\"_OPAREN__OPAREN_workflow.parameters.body_CPAREN__CPAREN_\"" }, { "name": "headers", - "value": 'PLACEHOLDER_OPENworkflow.parameters.headersPLACEHOLDER_CLOSE' + "value": "\"_OPAREN__OPAREN_workflow.parameters.body_CPAREN__CPAREN_\"" } ] } }]]' manifests/cicd-pipeline.yaml - sed -e 's/PLACEHOLDER_OPEN/{{/g' -e 's/PLACEHOLDER_CLOSE/}}/g' manifests/cicd-pipeline.yaml > manifests/cicd-pipeline.yaml.tmp && mv manifests/cicd-pipeline.yaml.tmp manifests/cicd-pipeline.yaml yq -i 'select(.kind =="Sensor").spec.triggers[0].template.argoWorkflow.source.resource.spec.volumes += [{ "name": "argocd-cluster-secret", "secret": { - "secretName": "dev-cluster-argo-secret", + "secretName": "dev-cluster-argo-secret" } }]' manifests/cicd-pipeline.yaml + + sed -i 's/"_OPAREN__OPAREN_/{{/g; s/_CPAREN__CPAREN_"/}}/g' manifests/cicd-pipeline.yaml + fi m_count=$(git status -s | wc -l) if [ $m_count -ge 1 ]; then From f1d9129c4c470cc97044b8a7d89e2bc0c538aec7 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Fri, 18 Apr 2025 17:07:49 -0700 Subject: [PATCH 39/63] final fixes --- .../skeleton/manifests/cicd-pipeline.yaml | 8 +- .../skeleton/manifests/workflow-rbac.yaml | 1 + .../skeleton/manifests/exit-handler.yaml | 96 ++++++++++--------- .../skeleton/manifests/wf-templates.yaml | 5 +- .../infra/terraform/setup-environments.sh | 12 ++- 5 files changed, 72 insertions(+), 50 deletions(-) diff --git a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml index 22ead44a6..06e3595ab 100644 --- a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/cicd-pipeline.yaml @@ -62,9 +62,9 @@ spec: - name: appname value: ${{values.appname}} - name: body - value: "" + value: "{{ .Input.body }}" - name: headers - value: "" + value: "{{ .Input.header }}" volumeClaimTemplates: - metadata: name: workdir @@ -104,11 +104,11 @@ spec: dest: spec.arguments.parameters.2.value - src: dependencyName: source-dep - dataTemplate: "{{ .Input.body }}" + dataKey: body dest: spec.arguments.parameters.7.value - src: dependencyName: source-dep - dataTemplate: "{{ .Input.header }}" + dataKey: header dest: spec.arguments.parameters.8.value --- apiVersion: v1 diff --git a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/workflow-rbac.yaml b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/workflow-rbac.yaml index 153055c88..5532d3962 100644 --- a/platform/backstage/templates/cicd-pipeline/skeleton/manifests/workflow-rbac.yaml +++ b/platform/backstage/templates/cicd-pipeline/skeleton/manifests/workflow-rbac.yaml @@ -50,6 +50,7 @@ rules: - get - list - watch + - patch - apiGroups: - "" resources: diff --git a/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml b/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml index c9dc6c0bb..262289851 100644 --- a/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml +++ b/platform/backstage/templates/dora-onboarding/skeleton/manifests/exit-handler.yaml @@ -5,7 +5,6 @@ metadata: namespace: ${{values.namespace}} spec: entrypoint: handle-rollout-check-and-process-deploy - serviceAccountName: ${{values.appname}}-dora-webhook-sa volumes: - name: argocd-cluster-secret secret: @@ -20,18 +19,12 @@ spec: templates: - name: handle-rollout-check-and-process-deploy steps: - - - name: fetch-argo-plugin - template: download-argo-plugin - - - name: check-rollout template: check-rollout-status arguments: parameters: - name: namespace value: team-"{{workflow.parameters.appname}}" - artifacts: - - name: rollout_plugin - from: "{{steps.fetch-argo-plugin.outputs.artifacts.rollout_plugin}}" - - name: process-and-send-deployment template: process-deployment @@ -64,12 +57,12 @@ spec: name: devlake-webhook-id source: | apk add --no-cache jq curl - RESULT="{{inputs.parameters.status}} + RESULT="{{inputs.parameters.status}}" webhook_url="{{inputs.parameters.url}}"/api/rest/plugins/webhook/connections/${DEVLAKE_HOOK_ID}/deployments PAYLOAD=$(echo '{{inputs.parameters.body}}' | jq --arg result "$RESULT" ' . as $root | { - id: .after, + id: ("deployment-" + (.id | .[0:7])), createdDate: (.commits | first | .timestamp), startedDate: (.commits | first | .timestamp), finishedDate: (.commits | last | .timestamp), @@ -85,7 +78,6 @@ spec: commitSha: .id, commitMsg: .message, result: "SUCCESS", - name: ("deployment-" + (.id | .[0:7])) } ] }') @@ -94,28 +86,10 @@ spec: curl -X POST -H "Authorization: Bearer $DEVLAKE_TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD" $webhook_url - - name: download-argo-plugin - container: - image: alpine/curl:8.12.1 - command: [/bin/bash, -c] - args: - - | - curl -LO https://github.com/argoproj/argo-rollouts/releases/latest/download/kubectl-argo-rollouts-linux-amd64 - chmod +x ./kubectl-argo-rollouts-linux-amd64 - mv ./kubectl-argo-rollouts-linux-amd64 > /tmp/kubectl-argo-rollouts-linux-amd64 - outputs: - artifacts: - - name: rollout_plugin - path: /tmp/kubectl-argo-rollouts-linux-amd64 - - - name: check-rollout-status inputs: parameters: - name: namespace - artifacts: - - name: rollout_plugin - path: /tmp/kubectl-argo-rollouts-linux-amd64 container: image: bitnami/kubectl:latest command: [/bin/bash, -c] @@ -126,27 +100,63 @@ spec: args: - | mkdir -p ~/.kube - cp /tmp/argocd-cluster/config ~/.kube/config - - mv /tmp/kubectl-argo-rollouts-linux-amd64 /usr/local/bin/kubectl-argo-rollouts + + # Read the secret components + SERVER=$(cat /tmp/argocd-cluster/server) + NAME=$(cat /tmp/argocd-cluster/name) + TOKEN=$(cat /tmp/argocd-cluster/config | jq -r .bearerToken) + CA_DATA=$(cat /tmp/argocd-cluster/config | jq -r .tlsClientConfig.caData) + + # Create kubeconfig + cat > ~/.kube/config < Date: Fri, 18 Apr 2025 17:13:05 -0700 Subject: [PATCH 40/63] init devlake with data --- .../infra/terraform/mgmt/terraform/devlake.tf | 16 +++++++++++++++- .../terraform/scripts/devlake/devlake-init.sh | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 8df18c158..9983203a4 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -122,9 +122,23 @@ resource "kubectl_manifest" "application_argocd_devlake" { yaml_body = templatefile("${path.module}/templates/argocd-apps/devlake.yaml", { GITHUB_URL = local.repo_url - GITHUB_BRANCH = "feat/dora-metrics" #TODO:SWITCH BEFORE PR + GITHUB_BRANCH = local.repo_branch } ) } +resource "terraform_data" "devlake_init_scripts" { + depends_on = [ + kubectl_manifest.application_argocd_devlake + ] + + provisioner "local-exec" { + command = "./devlake-init.sh ${local.domain_name}" + + working_dir = "${path.module}/scripts/devlake" + interpreter = ["/bin/bash", "-c"] + } + +} + diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh index d6b88fc3a..760a20f3c 100755 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh @@ -1,8 +1,8 @@ #!/bin/bash kubectl wait --for=jsonpath=.status.health.status=Healthy -n argocd application/devlake -kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=60s -kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=60s +kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=600s +kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=600s kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & pid=$! From 3821a709ce31bd04bc88508f31b147dc225e9c64 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Fri, 18 Apr 2025 17:16:19 -0700 Subject: [PATCH 41/63] revert hardcode --- platform/backstage/templates/catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/backstage/templates/catalog-info.yaml b/platform/backstage/templates/catalog-info.yaml index aab3994f8..98fbf99d8 100644 --- a/platform/backstage/templates/catalog-info.yaml +++ b/platform/backstage/templates/catalog-info.yaml @@ -35,4 +35,4 @@ metadata: description: Holds system information i.e, hostname, IP, OS, etc spec: owner: guest - hostname: modern-engg-52a778290c6c8c1f.elb.us-west-2.amazonaws.com + hostname: HOSTNAME From 140d97913ae68401fa39cf40b307c01e4515e1f3 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 21 Apr 2025 16:50:28 -0700 Subject: [PATCH 42/63] script fixes --- .../infra/terraform/destroy-environments.sh | 2 ++ .../infra/terraform/mgmt/terraform/devlake.tf | 28 +++++++++---------- .../terraform/scripts/devlake/devlake-init.sh | 3 +- .../infra/terraform/setup-environments.sh | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/platform/infra/terraform/destroy-environments.sh b/platform/infra/terraform/destroy-environments.sh index ae5c2d2fb..443005857 100755 --- a/platform/infra/terraform/destroy-environments.sh +++ b/platform/infra/terraform/destroy-environments.sh @@ -174,6 +174,8 @@ aws elbv2 delete-load-balancer --region $TF_VAR_aws_region --load-balancer-arn $ # Cleanup the keycloak and AMG Secrets config if not cleaned aws secretsmanager delete-secret --secret-id "modern-engg/keycloak/config" --force-delete-without-recovery --region $TF_VAR_aws_region || true +aws secretsmanager delete-secret --secret-id "modern-engg/devlake/encryption" --force-delete-without-recovery --region $TF_VAR_aws_region || true + aws secretsmanager delete-secret --secret-id "modern-engg/amg" --force-delete-without-recovery --region $TF_VAR_aws_region || true aws secretsmanager delete-secret --secret-id "/platform/amp" --force-delete-without-recovery --region $TF_VAR_aws_region || true diff --git a/platform/infra/terraform/mgmt/terraform/devlake.tf b/platform/infra/terraform/mgmt/terraform/devlake.tf index 9983203a4..5cc61d125 100644 --- a/platform/infra/terraform/mgmt/terraform/devlake.tf +++ b/platform/infra/terraform/mgmt/terraform/devlake.tf @@ -128,17 +128,17 @@ resource "kubectl_manifest" "application_argocd_devlake" { } -resource "terraform_data" "devlake_init_scripts" { - depends_on = [ - kubectl_manifest.application_argocd_devlake - ] - - provisioner "local-exec" { - command = "./devlake-init.sh ${local.domain_name}" - - working_dir = "${path.module}/scripts/devlake" - interpreter = ["/bin/bash", "-c"] - } - -} - +# resource "terraform_data" "devlake_init_scripts" { +# depends_on = [ +# kubectl_manifest.application_argocd_devlake +# ] +# +# provisioner "local-exec" { +# command = "./devlake-init.sh ${local.domain_name}" +# +# working_dir = "${path.module}/scripts/devlake" +# interpreter = ["/bin/bash", "-c"] +# } +# +# } +# diff --git a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh index 760a20f3c..a3b46f658 100755 --- a/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh +++ b/platform/infra/terraform/mgmt/terraform/scripts/devlake/devlake-init.sh @@ -1,8 +1,7 @@ #!/bin/bash kubectl wait --for=jsonpath=.status.health.status=Healthy -n argocd application/devlake -kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=600s -kubectl wait --for=condition=ready pod -l devlakeComponent=mysql -n devlake --timeout=600s +kubectl wait --for=condition=ready pod -l devlakeComponent=lake -n devlake --timeout=900s kubectl port-forward -n devlake svc/devlake-lake 9090:8080 >/dev/null 2>&1 & pid=$! diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 361c686fd..4ad871333 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -313,7 +313,7 @@ while true; do if [ -n "$CONN_SECRET_NAME" ]; then break fi - echo "Waiting for secret to be available..." + echo "Waiting for devlake connection secret to be available..." sleep 5 done export DEVLAKE_MYSQL_ENDPOINT=$(kubectl get secret ${CONN_SECRET_NAME} -n crossplane-system -o jsonpath='{.data.reader_endpoint}' | base64 --decode) From 314ebda7bf9e86ae3d5566dd03515acda1165b48 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Mon, 21 Apr 2025 17:19:52 -0700 Subject: [PATCH 43/63] final fixes --- packages/argo-rollouts/dev/cm-notifs.yaml | 18 ------------------ packages/argo-rollouts/dev/test-rollout.yaml | 8 -------- packages/devlake/dev/kustomization.yaml | 2 -- .../devlake/dev/patches/devlake-patch-sa.yaml | 9 --------- 4 files changed, 37 deletions(-) delete mode 100644 packages/argo-rollouts/dev/cm-notifs.yaml delete mode 100644 packages/argo-rollouts/dev/test-rollout.yaml delete mode 100644 packages/devlake/dev/patches/devlake-patch-sa.yaml diff --git a/packages/argo-rollouts/dev/cm-notifs.yaml b/packages/argo-rollouts/dev/cm-notifs.yaml deleted file mode 100644 index 19e9cbec7..000000000 --- a/packages/argo-rollouts/dev/cm-notifs.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: argo-rollouts-notification-configmap - namespace: argo-rollouts -data: - trigger.on-rollout-aborted: | - - send: [modengg-rollout-aborted] - trigger.on-rollout-completed: | - - send: [modengg-rollout-completed] - service.webhook.deployments: | - url: https://${DNS_ENGINEERING}/argo-events/deployments - template.modengg-rollout-aborted: | - - stuff - template.modengg-rollout-completed: | - - stuff - - diff --git a/packages/argo-rollouts/dev/test-rollout.yaml b/packages/argo-rollouts/dev/test-rollout.yaml deleted file mode 100644 index 75f0b1973..000000000 --- a/packages/argo-rollouts/dev/test-rollout.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: Rollout -metadata: - name: test-rollout - annotations: - notifications.argoproj.io/subscribe.on-rollout-completed.deployments: "" - notifications.argoproj.io/subscribe.on-rollout-aborted.deployments: "" diff --git a/packages/devlake/dev/kustomization.yaml b/packages/devlake/dev/kustomization.yaml index 58be0507c..6fb5a539a 100644 --- a/packages/devlake/dev/kustomization.yaml +++ b/packages/devlake/dev/kustomization.yaml @@ -4,5 +4,3 @@ resources: - external-secrets.yaml - mysql-init-job.yaml - rds-mysql.yaml -# patches: -# - path: patches/devlake-patch-sa.yaml diff --git a/packages/devlake/dev/patches/devlake-patch-sa.yaml b/packages/devlake/dev/patches/devlake-patch-sa.yaml deleted file mode 100644 index 0a6a3f300..000000000 --- a/packages/devlake/dev/patches/devlake-patch-sa.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: devlake-lake - namespace: devlake -spec: - template: - spec: - serviceAccountName: devlake-mysql-access From 21f034f69450d28a3b378b4deb951f1e92692d42 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 22 Apr 2025 19:19:36 -0700 Subject: [PATCH 44/63] change to env selector and yaml.gen --- platform/crossplane/compositions/rds/mysql-aurora.yaml | 10 +++++++--- .../mgmt/terraform/scripts/crossplane/.gitignore | 2 +- .../scripts/crossplane/cp-mgmt-env-config.yaml | 2 ++ platform/infra/terraform/setup-environments.sh | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml index f233f1e50..5a6f3acf7 100644 --- a/platform/crossplane/compositions/rds/mysql-aurora.yaml +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -12,9 +12,13 @@ metadata: spec: environment: environmentConfigs: - - type: Reference - ref: - name: mgmt-environment + - type: Selector + selector: + mode: Single + matchLabels: + - key: env + type: FromCompositeFieldPath + valueFromFieldPath: spec.parameters.deploy writeConnectionSecretsToNamespace: crossplane-system compositeTypeRef: apiVersion: awsblueprints.io/v1alpha1 diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore index eed59f21e..f227a4e36 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore @@ -1,2 +1,2 @@ -*.yml +*.yaml.gen diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml index fa4ef26ba..67928f0c8 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml @@ -2,6 +2,8 @@ apiVersion: apiextensions.crossplane.io/v1alpha1 kind: EnvironmentConfig metadata: name: mgmt-environment + labels: + env: mgmt data: locations: us: us-west-2 diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 4ad871333..3144d1893 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -72,8 +72,8 @@ DB_SUBNET_ID_2=$(terraform output -json database_subnet_ids | jq -r '.[1]') DB_SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') # Setup CP Env Config for MGMT Cluster -sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$DB_SUBNET_ID_1#${DB_SUBNET_ID_1}#g" -e "s#\$DB_SUBNET_ID_2#${DB_SUBNET_ID_2}#g" -e "s#\$DB_SUBNET_ID_3#${DB_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml -kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml +sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$DB_SUBNET_ID_1#${DB_SUBNET_ID_1}#g" -e "s#\$DB_SUBNET_ID_2#${DB_SUBNET_ID_2}#g" -e "s#\$DB_SUBNET_ID_3#${DB_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml.gen +kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml.gen export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) From 0ab29b31558a64ee1d07e0ff2840d42301fd3e26 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Tue, 22 Apr 2025 19:29:51 -0700 Subject: [PATCH 45/63] get mysql envconfig example --- .../base/provider-aws.yaml | 11 ++ .../compositions/rds/mysql-aurora.yaml | 165 ++++++++++++++++++ .../mgmt/terraform/mgmt-cluster/outputs.tf | 10 ++ .../mgmt/terraform/mgmt-cluster/vpc.tf | 9 +- .../terraform/scripts/crossplane/.gitignore | 3 +- .../crossplane/cp-mgmt-env-config.yaml | 6 + .../argocd-apps/crossplane-compositions.yaml | 2 + .../infra/terraform/setup-environments.sh | 9 +- 8 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 platform/crossplane/compositions/rds/mysql-aurora.yaml diff --git a/packages/crossplane-provider/base/provider-aws.yaml b/packages/crossplane-provider/base/provider-aws.yaml index 678325ab1..f5da7d43e 100644 --- a/packages/crossplane-provider/base/provider-aws.yaml +++ b/packages/crossplane-provider/base/provider-aws.yaml @@ -83,3 +83,14 @@ spec: package: xpkg.upbound.io/upbound/provider-aws-rds:v1.13.1 runtimeConfigRef: name: aws-irsa-drc +--- +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-ec2 + annotations: + argocd.argoproj.io/sync-wave: "10" +spec: + package: xpkg.upbound.io/upbound/provider-aws-ec2:v1.13.1 + runtimeConfigRef: + name: aws-irsa-drc diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml new file mode 100644 index 000000000..5a6f3acf7 --- /dev/null +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -0,0 +1,165 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: rds-aurora-mysql.awsblueprints.io + labels: + awsblueprints.io/provider: aws + awsblueprints.io/environment: prod + awsblueprints.io/createDBSubnetGroup: "true" +spec: + environment: + environmentConfigs: + - type: Selector + selector: + mode: Single + matchLabels: + - key: env + type: FromCompositeFieldPath + valueFromFieldPath: spec.parameters.deploy + writeConnectionSecretsToNamespace: crossplane-system + compositeTypeRef: + apiVersion: awsblueprints.io/v1alpha1 + kind: XRelationalDatabase + patchSets: + - name: common-fields + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.resourceConfig.deletionPolicy + toFieldPath: spec.deletionPolicy + - type: FromEnvironmentFieldPath + fromFieldPath: locations.us + toFieldPath: spec.forProvider.region + - name: connection-secret + patches: + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-mysql" + - fromFieldPath: "spec.writeConnectionSecretToRef.namespace" + toFieldPath: "spec.writeConnectionSecretToRef.namespace" + resources: + - base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: SubnetGroup + spec: + forProvider: + description: "rds-mysql" + patches: + - type: PatchSet + patchSetName: common-fields + - type: FromEnvironmentFieldPath + fromFieldPath: "dbSubnetIds" + toFieldPath: spec.forProvider.subnetIds + - base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroup + spec: + forProvider: + description: "rds-mysql-sg" + patches: + - type: PatchSet + patchSetName: common-fields + - type: FromCompositeFieldPath + fromFieldPath: spec.IngressRules + toFieldPath: spec.forProvider.ingress + policy: + mergeOptions: + appendSlice: true + - fromFieldPath: "metadata.uid" + toFieldPath: "spec.forProvider.name" + transforms: + - type: string + string: + fmt: "rds-mysql-sg-%s" + - type: FromEnvironmentFieldPath + fromFieldPath: "vpcId" + toFieldPath: "spec.forProvider.vpcId" + - base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroupIngressRule + spec: + forProvider: + fromPort: 3306 + toPort: 3306 + ipProtocol: "tcp" + cidrIpv4: "10.0.0.0/8" + securityGroupIdSelector: + matchControllerRef: true + patches: + - type: PatchSet + patchSetName: common-fields + + - base: + apiVersion: rds.aws.upbound.io/v1beta2 + kind: Cluster + spec: + forProvider: + autoGeneratePassword: true + backupRetentionPeriod: 7 + copyTagsToSnapshot: true + engine: aurora-mysql + engineVersion: "8.0.mysql_aurora.3.05.2" + finalDBSnapshotIdentifier: "to-be-patched" + masterPasswordSecretRef: + key: password + namespace: crossplane-system + masterUsername: root + preferredBackupWindow: 02:00-03:00 + preferredMaintenanceWindow: sun:04:00-sun:05:00 + skipFinalSnapshot: true + storageEncrypted: true + dbSubnetGroupNameSelector: + matchControllerRef: true + vpcSecurityGroupIdSelector: + matchControllerRef: true + patches: + - type: PatchSet + patchSetName: common-fields + - fromFieldPath: spec.writeConnectionSecretToRef.namespace + toFieldPath: spec.writeConnectionSecretToRef.namespace + - fromFieldPath: "metadata.name" + toFieldPath: "spec.forProvider.masterPasswordSecretRef.name" + transforms: + - type: string + string: + fmt: "%s-cluster-mysql-master-password" + - fromFieldPath: "metadata.name" + toFieldPath: "spec.writeConnectionSecretToRef.name" + transforms: + - type: string + string: + fmt: "%s-cluster-mysql-connection" + - fromFieldPath: "metadata.name" + toFieldPath: "spec.forProvider.finalDBSnapshotIdentifier" + transforms: + - type: string + string: + fmt: "%s-final-snapshot" + - fromFieldPath: "spec.engineVersion" + toFieldPath: "spec.forProvider.engineVersion" + - fromFieldPath: "spec.databaseName" + toFieldPath: "spec.forProvider.databaseName" + connectionDetails: + - fromConnectionSecretKey: username + - fromConnectionSecretKey: password + - fromConnectionSecretKey: endpoint + - fromConnectionSecretKey: port + - base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: ClusterInstance + spec: + forProvider: + clusterIdentifierSelector: + matchControllerRef: true + dbSubnetGroupNameSelector: + matchControllerRef: true + instanceClass: db.r5.xlarge + engine: aurora-mysql + patches: + - type: PatchSet + patchSetName: common-fields diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf index fac829ea4..7297f36d5 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/outputs.tf @@ -8,11 +8,21 @@ output "eks_cluster_private_subnets" { value = module.vpc.private_subnets } +output "eks_cluster_node_security_group_id" { + description = "EKS Cluster Node Security Group Id" + value = module.eks.node_security_group_id +} + output "vpc_cidr" { description = "Default VPC CIDR of the VPC created by the Module" value = local.vpc_cidr } +output "database_subnet_ids" { + description = "VPC Database Subnet Ids" + value = module.vpc.database_subnets +} + output "availability_zones" { description = "Default Availability Zone of the VPC created by the Module" value = local.azs diff --git a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf index 1362ee3d2..94a84537b 100644 --- a/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf +++ b/platform/infra/terraform/mgmt/terraform/mgmt-cluster/vpc.tf @@ -7,11 +7,16 @@ module "vpc" { azs = local.azs private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + database_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k + 4)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k + 8)] enable_nat_gateway = true single_nat_gateway = true + create_database_subnet_route_table = true + create_database_internet_gateway_route = false + create_database_nat_gateway_route = true + public_subnet_tags = { "kubernetes.io/role/elb" = 1 } @@ -23,4 +28,4 @@ module "vpc" { } tags = local.tags -} \ No newline at end of file +} diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore index 1cda54be9..f227a4e36 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/.gitignore @@ -1 +1,2 @@ -*.yml +*.yaml.gen + diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml index c44ed240c..67928f0c8 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml @@ -2,6 +2,8 @@ apiVersion: apiextensions.crossplane.io/v1alpha1 kind: EnvironmentConfig metadata: name: mgmt-environment + labels: + env: mgmt data: locations: us: us-west-2 @@ -9,5 +11,9 @@ data: - $PRIV_SUBNET_ID_1 - $PRIV_SUBNET_ID_2 - $PRIV_SUBNET_ID_3 + dbSubnetIds: + - $DB_SUBNET_ID_1 + - $DB_SUBNET_ID_2 + - $DB_SUBNET_ID_3 vpcId: $VPC_ID diff --git a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml index 5afd3f73a..8f0823f57 100644 --- a/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml +++ b/platform/infra/terraform/mgmt/terraform/templates/argocd-apps/crossplane-compositions.yaml @@ -13,6 +13,8 @@ spec: repoURL: ${GITHUB_URL} targetRevision: ${GITHUB_BRANCH} path: "platform/crossplane/compositions" + directory: + recurse: true destination: server: "https://kubernetes.default.svc" namespace: crossplane-system diff --git a/platform/infra/terraform/setup-environments.sh b/platform/infra/terraform/setup-environments.sh index 2c07cfa71..a3ee2a995 100755 --- a/platform/infra/terraform/setup-environments.sh +++ b/platform/infra/terraform/setup-environments.sh @@ -58,6 +58,7 @@ export GITHUB_BRANCH=$(yq '.repo_branch' ${REPO_ROOT}/platform/infra/terraform/m # Deploy the apps on IDP Builder and ArgoCD ${REPO_ROOT}/platform/infra/terraform/mgmt/setups/install.sh + cd ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/mgmt-cluster/ VPC_ID=$(terraform output -raw eks_cluster_vpc_id) @@ -66,9 +67,13 @@ PRIV_SUBNET_ID_1=$(terraform output -json eks_cluster_private_subnets | jq -r '. PRIV_SUBNET_ID_2=$(terraform output -json eks_cluster_private_subnets | jq -r '.[1]') PRIV_SUBNET_ID_3=$(terraform output -json eks_cluster_private_subnets | jq -r '.[2]') +DB_SUBNET_ID_1=$(terraform output -json database_subnet_ids | jq -r '.[0]') +DB_SUBNET_ID_2=$(terraform output -json database_subnet_ids | jq -r '.[1]') +DB_SUBNET_ID_3=$(terraform output -json database_subnet_ids | jq -r '.[2]') + # Setup CP Env Config for MGMT Cluster -sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml -kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yml +sed -e "s#\$PRIV_SUBNET_ID_1#${PRIV_SUBNET_ID_1}#g" -e "s#\$PRIV_SUBNET_ID_2#${PRIV_SUBNET_ID_2}#g" -e "s#\$PRIV_SUBNET_ID_3#${PRIV_SUBNET_ID_3}#g" -e "s#\$DB_SUBNET_ID_1#${DB_SUBNET_ID_1}#g" -e "s#\$DB_SUBNET_ID_2#${DB_SUBNET_ID_2}#g" -e "s#\$DB_SUBNET_ID_3#${DB_SUBNET_ID_3}#g" -e "s#\$VPC_ID#${VPC_ID}#g" ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml >${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml.gen +kubectl apply -f ${REPO_ROOT}/platform/infra/terraform/mgmt/terraform/scripts/crossplane/cp-mgmt-env-config.yaml.gen export TF_eks_cluster_vpc_id=$(terraform output -raw eks_cluster_vpc_id) export TF_eks_cluster_private_subnets=$(terraform output -json eks_cluster_private_subnets) From c79a8711a991cd140f43585865cab26838795968 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 23 Apr 2025 16:45:38 -0700 Subject: [PATCH 46/63] add vela templates and add label to composition --- .../components/rds-cluster-mysql.yaml | 51 ++++++++++++++++++ platform/components/rds-cluster-mysql.cue | 53 +++++++++++++++++++ .../compositions/rds/mysql-aurora.yaml | 3 +- .../terraform/scripts/crossplane/README.md | 4 +- 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml create mode 100644 platform/components/rds-cluster-mysql.cue diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml new file mode 100644 index 000000000..9a4aca09c --- /dev/null +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -0,0 +1,51 @@ +apiVersion: core.oam.dev/v1beta1 +kind: ComponentDefinition +metadata: + annotations: + definition.oam.dev/alias: "" + definition.oam.dev/description: Amazon RDS MySQL Cluster + name: rds-cluster-mysql +spec: + schematic: + cue: + template: | + output: { + apiVersion: "awsblueprints.io/v1alpha1" + kind: "RelationalDatabase" + metadata: { + name: context.name + namespace: context.namespace + } + spec: { + compositionSelector: matchLabels: { + "awsblueprints.io/provider": "aws" + "awsblueprints.io/environment": "prod" + "awsblueprints.io/createDBSubnetGroup": "true" + "awsblueprints.io/dbEngine": "mysql" + } + resourceConfig: { + name: context.name + deletionPolicy: "Delete" + tags: [ + { + key: "env" + value: "prod" + }, + { + key: "app" + value: "\(context.name)" + }, + ] + providerConfigName: "default" + region: "us-west-2" + } + databaseName: "initdb" + deploy: "\(parameter.deploy)" + } + } + parameter: deploy: string + workload: + definition: + apiVersion: apps/v1 + kind: Deployment + diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue new file mode 100644 index 000000000..3a1c28ba6 --- /dev/null +++ b/platform/components/rds-cluster-mysql.cue @@ -0,0 +1,53 @@ +"rds-cluster-mysql": { + alias: "" + annotations: {} + attributes: workload: definition: { + apiVersion: "apps/v1" + kind: "Deployment" + } + description: "Amazon RDS MySQL Cluster" + labels: {} + type: "component" +} + +template: { + output: { + apiVersion: "awsblueprints.io/v1alpha1" + kind: "RelationalDatabase" + metadata: { + name: context.name + namespace: context.namespace + } + spec: { + compositionSelector: { + matchLabels: { + "awsblueprints.io/provider": "aws" + "awsblueprints.io/environment": "prod" + "awsblueprints.io/createDBSubnetGroup": "true" + "awsblueprints.io/dbEngine": "mysql" + } + } + resourceConfig: { + name: context.name + deletionPolicy: "Delete" + tags: [ + { + key: "env" + value: "prod" + }, + { + key: "app" + value: "\(context.name)" + }, + ] + providerConfigName: "default" + region: "us-west-2" + } + databaseName: "initdb" + deploy: "\(parameter.deploy)" + } + } + parameter: { + deploy: string + } +} diff --git a/platform/crossplane/compositions/rds/mysql-aurora.yaml b/platform/crossplane/compositions/rds/mysql-aurora.yaml index 5a6f3acf7..ac908ec94 100644 --- a/platform/crossplane/compositions/rds/mysql-aurora.yaml +++ b/platform/crossplane/compositions/rds/mysql-aurora.yaml @@ -9,6 +9,7 @@ metadata: awsblueprints.io/provider: aws awsblueprints.io/environment: prod awsblueprints.io/createDBSubnetGroup: "true" + awsblueprints.io/dbEngine: mysql spec: environment: environmentConfigs: @@ -18,7 +19,7 @@ spec: matchLabels: - key: env type: FromCompositeFieldPath - valueFromFieldPath: spec.parameters.deploy + valueFromFieldPath: spec.deploy writeConnectionSecretsToNamespace: crossplane-system compositeTypeRef: apiVersion: awsblueprints.io/v1alpha1 diff --git a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md index c464c6151..0321f11fa 100644 --- a/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md +++ b/platform/infra/terraform/mgmt/terraform/scripts/crossplane/README.md @@ -4,4 +4,6 @@ After VPC is created, the values for VPC Id and Subnet Ids in the environment co This enables any Crossplane XRDs created in the crossplane compositions on the management cluster to substitute these values into managed resources. This ensures that customers will not need to manually go in and specify these values when creating a resource using the XRDs. -There is an example in the `mysql-aurora` XRD merged in from the Measuring Platform Success feature branch. +The environment config to be used is selected by labels specified in the xrd, such as the `env` label being `mgmt`, `dev`, or `prod`, for each environment. + +There is an example in the `mysql-aurora` XRD and a usage of this in the `rds-cluster-mysql` vela template. From bd7e7787f9f5d69fc162de180aeef696ed347d13 Mon Sep 17 00:00:00 2001 From: Zach Jacobson Date: Wed, 23 Apr 2025 17:00:28 -0700 Subject: [PATCH 47/63] changes to fix environment config selection --- .../kubevela/templates/components/rds-cluster-mysql.yaml | 2 -- platform/components/rds-cluster-mysql.cue | 2 -- platform/crossplane/compositions/rds/definition.yaml | 5 +++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml index 9a4aca09c..745425a1e 100644 --- a/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml +++ b/deployment/addons/kubevela/templates/components/rds-cluster-mysql.yaml @@ -36,8 +36,6 @@ spec: value: "\(context.name)" }, ] - providerConfigName: "default" - region: "us-west-2" } databaseName: "initdb" deploy: "\(parameter.deploy)" diff --git a/platform/components/rds-cluster-mysql.cue b/platform/components/rds-cluster-mysql.cue index 3a1c28ba6..bf8f1f841 100644 --- a/platform/components/rds-cluster-mysql.cue +++ b/platform/components/rds-cluster-mysql.cue @@ -40,8 +40,6 @@ template: { value: "\(context.name)" }, ] - providerConfigName: "default" - region: "us-west-2" } databaseName: "initdb" deploy: "\(parameter.deploy)" diff --git a/platform/crossplane/compositions/rds/definition.yaml b/platform/crossplane/compositions/rds/definition.yaml index f4402e6ef..22a6cad71 100644 --- a/platform/crossplane/compositions/rds/definition.yaml +++ b/platform/crossplane/compositions/rds/definition.yaml @@ -27,6 +27,8 @@ spec: properties: spec: properties: + deploy: + type: string databaseName: type: string dbSubnetGroupName: @@ -90,9 +92,8 @@ spec: type: object type: array required: - - providerConfigName - - region - tags + - deploy type: object required: - resourceConfig From 886466994fed352d2d1099ee9237dbebcfdbfe14 Mon Sep 17 00:00:00 2001 From: Shardul Vaidya Date: Thu, 8 May 2025 15:29:10 -0400 Subject: [PATCH 48/63] Fixed NextJS UX issue where they loaded the website before starting the backend --- applications/next-js/Dockerfile | 24 +- applications/next-js/app/[page]/page.tsx | 14 +- applications/next-js/app/backend-down.tsx | 17 + applications/next-js/app/page.tsx | 6 +- .../next-js/app/product/[handle]/page.tsx | 39 +- .../next-js/app/search/[collection]/page.tsx | 19 +- applications/next-js/app/search/page.tsx | 26 +- applications/next-js/components/carousel.tsx | 19 +- .../next-js/components/cart/actions.ts | 36 +- .../next-js/components/cart/index.tsx | 13 +- .../next-js/components/grid/three-items.tsx | 36 +- .../next-js/components/layout/footer.tsx | 2 +- .../components/layout/navbar/index.tsx | 4 +- .../components/layout/search/collections.tsx | 2 +- .../next-js/components/wishlist/actions.ts | 26 +- .../next-js/components/wishlist/index.tsx | 13 +- applications/next-js/lib/dynamo/index.ts | 167 +- applications/next-js/package.json | 4 +- applications/next-js/pnpm-lock.yaml | 19601 ++++++++++------ 19 files changed, 12133 insertions(+), 7935 deletions(-) create mode 100644 applications/next-js/app/backend-down.tsx diff --git a/applications/next-js/Dockerfile b/applications/next-js/Dockerfile index 1fb1a3cd0..8a6b22529 100644 --- a/applications/next-js/Dockerfile +++ b/applications/next-js/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine AS base +FROM node:20-alpine AS base # Install dependencies only when needed FROM base AS deps @@ -9,11 +9,11 @@ WORKDIR /app # Install dependencies based on the preferred package manager COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./ RUN \ - if [ -f yarn.lock ]; then yarn --frozen-lockfile; \ - elif [ -f package-lock.json ]; then npm ci; \ - elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \ - else echo "Lockfile not found." && exit 1; \ - fi + if [ -f yarn.lock ]; then yarn --frozen-lockfile; \ + elif [ -f package-lock.json ]; then npm ci; \ + elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \ + else echo "Lockfile not found." && exit 1; \ + fi # Rebuild the source code only when needed @@ -28,11 +28,11 @@ COPY . . ENV NEXT_TELEMETRY_DISABLED=1 RUN \ - if [ -f yarn.lock ]; then yarn run build; \ - elif [ -f package-lock.json ]; then npm run build; \ - elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \ - else echo "Lockfile not found." && exit 1; \ - fi + if [ -f yarn.lock ]; then yarn run build; \ + elif [ -f package-lock.json ]; then npm run build; \ + elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \ + else echo "Lockfile not found." && exit 1; \ + fi # Production image, copy all the files and run next FROM base AS runner @@ -65,4 +65,4 @@ ENV PORT=8080 # server.js is created by next build from the standalone output # https://nextjs.org/docs/pages/api-reference/next-config-js/output ENV HOSTNAME="0.0.0.0" -CMD ["node", "server.js"] \ No newline at end of file +CMD ["node", "server.js"] diff --git a/applications/next-js/app/[page]/page.tsx b/applications/next-js/app/[page]/page.tsx index f7b8b4683..2bd265106 100644 --- a/applications/next-js/app/[page]/page.tsx +++ b/applications/next-js/app/[page]/page.tsx @@ -1,15 +1,11 @@ -import Prose from "components/prose"; -import { getPage } from "lib/dynamo"; -import { notFound } from "next/navigation"; +import Prose from 'components/prose'; +import { getPage } from 'lib/dynamo'; +import { notFound } from 'next/navigation'; export default async function Page({ params }: { params: { page: string } }) { const page = await getPage(params.page); - if (!page) return notFound(); + if (!page || page instanceof Error) return notFound(); - return ( - <> - - - ); + return <>; } diff --git a/applications/next-js/app/backend-down.tsx b/applications/next-js/app/backend-down.tsx new file mode 100644 index 000000000..7420502fb --- /dev/null +++ b/applications/next-js/app/backend-down.tsx @@ -0,0 +1,17 @@ +import React from 'react'; + +export default function NotFound() { + return ( +
+
+
+

500

+
+

+ Looks like the backend is down, please deploy it and try again! +

+
+
+
+ ); +} diff --git a/applications/next-js/app/page.tsx b/applications/next-js/app/page.tsx index bb329b6ba..aa3648cf1 100644 --- a/applications/next-js/app/page.tsx +++ b/applications/next-js/app/page.tsx @@ -1,6 +1,6 @@ -import Footer from "components/layout/footer"; -import { Carousel } from "components/carousel"; -import { ThreeItemGrid } from "components/grid/three-items"; +import Footer from 'components/layout/footer'; +import { Carousel } from 'components/carousel'; +import { ThreeItemGrid } from 'components/grid/three-items'; export default async function HomePage() { return ( diff --git a/applications/next-js/app/product/[handle]/page.tsx b/applications/next-js/app/product/[handle]/page.tsx index 048f89701..54a2f8023 100644 --- a/applications/next-js/app/product/[handle]/page.tsx +++ b/applications/next-js/app/product/[handle]/page.tsx @@ -1,30 +1,27 @@ -import { notFound } from "next/navigation"; +import { notFound } from 'next/navigation'; -import { GridTileImage } from "components/grid/tile"; -import Footer from "components/layout/footer"; -import { Gallery } from "components/product/gallery"; -import { ProductDescription } from "components/product/product-description"; -import { HIDDEN_PRODUCT_TAG } from "lib/constants"; -import { getProduct } from "lib/dynamo"; -import { Image } from "lib/dynamo/types"; -import Link from "next/link"; -import { Suspense } from "react"; +import { GridTileImage } from 'components/grid/tile'; +import Footer from 'components/layout/footer'; +import { Gallery } from 'components/product/gallery'; +import { ProductDescription } from 'components/product/product-description'; +import { HIDDEN_PRODUCT_TAG } from 'lib/constants'; +import { getProduct } from 'lib/dynamo'; +import { Image } from 'lib/dynamo/types'; +import Link from 'next/link'; +import { Suspense } from 'react'; +import NotFound from 'app/backend-down'; -export default async function ProductPage({ - params, -}: { - params: { handle: string }; -}) { +export default async function ProductPage({ params }: { params: { handle: string } }) { const product = await getProduct(params.handle); - if (!product) return notFound(); + if (!product || product instanceof Error) return NotFound(); const productJsonLd = { - "@context": "https://schema.org", - "@type": "Product", + '@context': 'https://schema.org', + '@type': 'Product', name: product.name, description: product.description, - image: product.images, + image: product.images }; return ( @@ -32,7 +29,7 @@ export default async function ProductPage({