Merge branch 'ClusterCockpit:master' into master

Author: oscarminus

cmd/cc-backend/cli.go

@@ -12,7 +12,7 @@ var (
 )
 
 func cliInit() {
-	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize swlite database file, config.json and .env")
+	flag.BoolVar(&flagInit, "init", false, "Setup var directory, initialize sqlite database file, config.json and .env")
 	flag.BoolVar(&flagReinitDB, "init-db", false, "Go through job-archive and re-initialize the 'job', 'tag', and 'jobtag' tables (all running jobs will be lost!)")
 	flag.BoolVar(&flagSyncLDAP, "sync-ldap", false, "Sync the 'hpc_user' table with ldap")
 	flag.BoolVar(&flagServer, "server", false, "Start a server, continues listening on port after initialization and argument handling")
@@ -24,10 +24,10 @@ func cliInit() {
 	flag.BoolVar(&flagForceDB, "force-db", false, "Force database version, clear dirty flag and exit")
 	flag.BoolVar(&flagLogDateTime, "logdate", false, "Set this flag to add date and time to log messages")
 	flag.StringVar(&flagConfigFile, "config", "./config.json", "Specify alternative path to `config.json`")
-	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: `<username>:[admin,support,manager,api,user]:<password>`")
-	flag.StringVar(&flagDelUser, "del-user", "", "Remove user by `username`")
+	flag.StringVar(&flagNewUser, "add-user", "", "Add a new user. Argument format: <username>:[admin,support,manager,api,user]:<password>")
+	flag.StringVar(&flagDelUser, "del-user", "", "Remove a existing user. Argument format: <username>")
 	flag.StringVar(&flagGenJWT, "jwt", "", "Generate and print a JWT for the user specified by its `username`")
 	flag.StringVar(&flagImportJob, "import-job", "", "Import a job. Argument format: `<path-to-meta.json>:<path-to-data.json>,...`")
-	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug,info,warn (default),err,fatal,crit]`")
+	flag.StringVar(&flagLogLevel, "loglevel", "warn", "Sets the logging level: `[debug, info (default), warn, err, crit]`")
 	flag.Parse()
 }

cmd/cc-backend/init.go

@@ -5,7 +5,6 @@
 package main
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/ClusterCockpit/cc-backend/internal/repository"
@@ -62,24 +61,23 @@ const configString = `
 
 func initEnv() {
 	if util.CheckFileExists("var") {
-		fmt.Print("Directory ./var already exists. Exiting!\n")
-		os.Exit(0)
+		log.Exit("Directory ./var already exists. Cautiously exiting application initialization.")
 	}
 
 	if err := os.WriteFile("config.json", []byte(configString), 0o666); err != nil {
-		log.Fatalf("Writing config.json failed: %s", err.Error())
+		log.Abortf("Could not write default ./config.json with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 
 	if err := os.WriteFile(".env", []byte(envString), 0o666); err != nil {
-		log.Fatalf("Writing .env failed: %s", err.Error())
+		log.Abortf("Could not write default ./.env file with permissions '0o666'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 
 	if err := os.Mkdir("var", 0o777); err != nil {
-		log.Fatalf("Mkdir var failed: %s", err.Error())
+		log.Abortf("Could not create default ./var folder with permissions '0o777'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 
 	err := repository.MigrateDB("sqlite3", "./var/job.db")
 	if err != nil {
-		log.Fatalf("Initialize job.db failed: %s", err.Error())
+		log.Abortf("Could not initialize default sqlite3 database as './var/job.db'. Application initialization failed, exited.\nError: %s\n", err.Error())
 	}
 }

cmd/cc-backend/main.go

@@ -61,15 +61,23 @@ func main() {
 	// Apply config flags for pkg/log
 	log.Init(flagLogLevel, flagLogDateTime)
 
+	// If init flag set, run tasks here before any file dependencies cause errors
+	if flagInit {
+		initEnv()
+		log.Exit("Successfully setup environment!\n" +
+			"Please review config.json and .env and adjust it to your needs.\n" +
+			"Add your job-archive at ./var/job-archive.")
+	}
+
 	// See https://github.com/google/gops (Runtime overhead is almost zero)
 	if flagGops {
 		if err := agent.Listen(agent.Options{}); err != nil {
-			log.Fatalf("gops/agent.Listen failed: %s", err.Error())
+			log.Abortf("Could not start gops agent with 'gops/agent.Listen(agent.Options{})'. Application startup failed, exited.\nError: %s\n", err.Error())
 		}
 	}
 
 	if err := runtimeEnv.LoadEnv("./.env"); err != nil && !os.IsNotExist(err) {
-		log.Fatalf("parsing './.env' file failed: %s", err.Error())
+		log.Abortf("Could not parse existing .env file at location './.env'. Application startup failed, exited.\nError: %s\n", err.Error())
 	}
 
 	// Initialize sub-modules and handle command line flags.
@@ -87,119 +95,122 @@ func main() {
 	if flagMigrateDB {
 		err := repository.MigrateDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			log.Abortf("MigrateDB Failed: Could not migrate '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error())
 		}
-		os.Exit(0)
+		log.Exitf("MigrateDB Success: Migrated '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version)
 	}
 
 	if flagRevertDB {
 		err := repository.RevertDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			log.Abortf("RevertDB Failed: Could not revert '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1), err.Error())
 		}
-		os.Exit(0)
+		log.Exitf("RevertDB Success: Reverted '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, (repository.Version - 1))
 	}
 
 	if flagForceDB {
 		err := repository.ForceDB(config.Keys.DBDriver, config.Keys.DB)
 		if err != nil {
-			log.Fatal(err)
+			log.Abortf("ForceDB Failed: Could not force '%s' database at location '%s' to version %d.\nError: %s\n", config.Keys.DBDriver, config.Keys.DB, repository.Version, err.Error())
 		}
-		os.Exit(0)
+		log.Exitf("ForceDB Success: Forced '%s' database at location '%s' to version %d.\n", config.Keys.DBDriver, config.Keys.DB, repository.Version)
 	}
 
 	repository.Connect(config.Keys.DBDriver, config.Keys.DB)
 
-	if flagInit {
-		initEnv()
-		fmt.Print("Successfully setup environment!\n")
-		fmt.Print("Please review config.json and .env and adjust it to your needs.\n")
-		fmt.Print("Add your job-archive at ./var/job-archive.\n")
-		os.Exit(0)
-	}
-
 	if !config.Keys.DisableAuthentication {
 
 		auth.Init()
 
 		if flagNewUser != "" {
 			parts := strings.SplitN(flagNewUser, ":", 3)
 			if len(parts) != 3 || len(parts[0]) == 0 {
-				log.Fatal("invalid argument format for user creation")
+				log.Abortf("Add User: Could not parse supplied argument format: No changes.\n"+
+					"Want: <username>:[admin,support,manager,api,user]:<password>\n"+
+					"Have: %s\n", flagNewUser)
 			}
 
 			ur := repository.GetUserRepository()
 			if err := ur.AddUser(&schema.User{
 				Username: parts[0], Projects: make([]string, 0), Password: parts[2], Roles: strings.Split(parts[1], ","),
 			}); err != nil {
-				log.Fatalf("adding '%s' user authentication failed: %v", parts[0], err)
+				log.Abortf("Add User: Could not add new user authentication for '%s' and roles '%s'.\nError: %s\n", parts[0], parts[1], err.Error())
+			} else {
+				log.Printf("Add User: Added new user '%s' with roles '%s'.\n", parts[0], parts[1])
 			}
 		}
+
 		if flagDelUser != "" {
 			ur := repository.GetUserRepository()
 			if err := ur.DelUser(flagDelUser); err != nil {
-				log.Fatalf("deleting user failed: %v", err)
+				log.Abortf("Delete User: Could not delete user '%s' from DB.\nError: %s\n", flagDelUser, err.Error())
+			} else {
+				log.Printf("Delete User: Deleted user '%s' from DB.\n", flagDelUser)
 			}
 		}
 
 		authHandle := auth.GetAuthInstance()
 
 		if flagSyncLDAP {
 			if authHandle.LdapAuth == nil {
-				log.Fatal("cannot sync: LDAP authentication is not configured")
+				log.Abort("Sync LDAP: LDAP authentication is not configured, could not synchronize. No changes, exited.")
 			}
 
 			if err := authHandle.LdapAuth.Sync(); err != nil {
-				log.Fatalf("LDAP sync failed: %v", err)
+				log.Abortf("Sync LDAP: Could not synchronize, failed with error.\nError: %s\n", err.Error())
 			}
-			log.Info("LDAP sync successfull")
+			log.Print("Sync LDAP: LDAP synchronization successfull.")
 		}
 
 		if flagGenJWT != "" {
 			ur := repository.GetUserRepository()
 			user, err := ur.GetUser(flagGenJWT)
 			if err != nil {
-				log.Fatalf("could not get user from JWT: %v", err)
+				log.Abortf("JWT: Could not get supplied user '%s' from DB. No changes, exited.\nError: %s\n", flagGenJWT, err.Error())
 			}
 
 			if !user.HasRole(schema.RoleApi) {
-				log.Warnf("user '%s' does not have the API role", user.Username)
+				log.Warnf("JWT: User '%s' does not have the role 'api'. REST API endpoints will return error!\n", user.Username)
 			}
 
 			jwt, err := authHandle.JwtAuth.ProvideJWT(user)
 			if err != nil {
-				log.Fatalf("failed to provide JWT to user '%s': %v", user.Username, err)
+				log.Abortf("JWT: User '%s' found in DB, but failed to provide JWT.\nError: %s\n", user.Username, err.Error())
 			}
 
-			fmt.Printf("MAIN > JWT for '%s': %s\n", user.Username, jwt)
+			log.Printf("JWT: Successfully generated JWT for user '%s': %s\n", user.Username, jwt)
 		}
 
 	} else if flagNewUser != "" || flagDelUser != "" {
-		log.Fatal("arguments --add-user and --del-user can only be used if authentication is enabled")
+		log.Abort("Error: Arguments '--add-user' and '--del-user' can only be used if authentication is enabled. No changes, exited.")
 	}
 
 	if err := archive.Init(config.Keys.Archive, config.Keys.DisableArchive); err != nil {
-		log.Fatalf("failed to initialize archive: %s", err.Error())
+		log.Abortf("Init: Failed to initialize archive.\nError: %s\n", err.Error())
 	}
 
 	if err := metricdata.Init(); err != nil {
-		log.Fatalf("failed to initialize metricdata repository: %s", err.Error())
+		log.Abortf("Init: Failed to initialize metricdata repository.\nError %s\n", err.Error())
 	}
 
 	if flagReinitDB {
 		if err := importer.InitDB(); err != nil {
-			log.Fatalf("failed to re-initialize repository DB: %s", err.Error())
+			log.Abortf("Init DB: Failed to re-initialize repository DB.\nError: %s\n", err.Error())
+		} else {
+			log.Print("Init DB: Sucessfully re-initialized repository DB.")
 		}
 	}
 
 	if flagImportJob != "" {
 		if err := importer.HandleImportFlag(flagImportJob); err != nil {
-			log.Fatalf("job import failed: %s", err.Error())
+			log.Abortf("Import Job: Job import failed.\nError: %s\n", err.Error())
+		} else {
+			log.Printf("Import Job: Imported Job '%s' into DB.\n", flagImportJob)
 		}
 	}
 
 	if !flagServer {
-		return
+		log.Exit("No errors, server flag not set. Exiting cc-backend.")
 	}
 
 	archiver.Start(repository.GetJobRepository())

cmd/cc-backend/server.go

@@ -64,7 +64,7 @@ func serverInit() {
 			case string:
 				return fmt.Errorf("MAIN > Panic: %s", e)
 			case error:
-				return fmt.Errorf("MAIN > Panic caused by: %w", e)
+				return fmt.Errorf("MAIN > Panic caused by: %s", e.Error())
 			}
 
 			return errors.New("MAIN > Internal server error (panic)")
@@ -268,7 +268,7 @@ func serverStart() {
 	// Start http or https server
 	listener, err := net.Listen("tcp", config.Keys.Addr)
 	if err != nil {
-		log.Fatalf("starting http listener failed: %v", err)
+		log.Abortf("Server Start: Starting http listener on '%s' failed.\nError: %s\n", config.Keys.Addr, err.Error())
 	}
 
 	if !strings.HasSuffix(config.Keys.Addr, ":80") && config.Keys.RedirectHttpTo != "" {
@@ -281,7 +281,7 @@ func serverStart() {
 		cert, err := tls.LoadX509KeyPair(
 			config.Keys.HttpsCertFile, config.Keys.HttpsKeyFile)
 		if err != nil {
-			log.Fatalf("loading X509 keypair failed: %v", err)
+			log.Abortf("Server Start: Loading X509 keypair failed. Check options 'https-cert-file' and 'https-key-file' in 'config.json'.\nError: %s\n", err.Error())
 		}
 		listener = tls.NewListener(listener, &tls.Config{
 			Certificates: []tls.Certificate{cert},
@@ -292,20 +292,20 @@ func serverStart() {
 			MinVersion:               tls.VersionTLS12,
 			PreferServerCipherSuites: true,
 		})
-		fmt.Printf("HTTPS server listening at %s...", config.Keys.Addr)
+		log.Printf("HTTPS server listening at %s...\n", config.Keys.Addr)
 	} else {
-		fmt.Printf("HTTP server listening at %s...", config.Keys.Addr)
+		log.Printf("HTTP server listening at %s...\n", config.Keys.Addr)
 	}
 	//
 	// Because this program will want to bind to a privileged port (like 80), the listener must
 	// be established first, then the user can be changed, and after that,
 	// the actual http server can be started.
 	if err := runtimeEnv.DropPrivileges(config.Keys.Group, config.Keys.User); err != nil {
-		log.Fatalf("error while preparing server start: %s", err.Error())
+		log.Abortf("Server Start: Error while preparing server start.\nError: %s\n", err.Error())
 	}
 
 	if err = server.Serve(listener); err != nil && err != http.ErrServerClosed {
-		log.Fatalf("starting server failed: %v", err)
+		log.Abortf("Server Start: Starting server failed.\nError: %s\n", err.Error())
 	}
 }
 

go.mod

@@ -1,6 +1,7 @@
 module github.com/ClusterCockpit/cc-backend
 
 go 1.23.5
+toolchain go1.24.1
 
 require (
 	github.com/99designs/gqlgen v0.17.66
@@ -10,7 +11,7 @@ require (
 	github.com/go-co-op/gocron/v2 v2.16.0
 	github.com/go-ldap/ldap/v3 v3.4.10
 	github.com/go-sql-driver/mysql v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/golang-migrate/migrate/v4 v4.18.2
 	github.com/google/gops v0.3.28
 	github.com/gorilla/handlers v1.5.2
@@ -78,7 +79,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/net v0.36.0 // indirect
 	golang.org/x/sync v0.11.0 // indirect
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/text v0.22.0 // indirect

go.sum

@@ -83,8 +83,8 @@ github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIx
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
+github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-migrate/migrate/v4 v4.18.2 h1:2VSCMz7x7mjyTXx3m2zPokOY82LTRgxK1yQYKo6wWQ8=
 github.com/golang-migrate/migrate/v4 v4.18.2/go.mod h1:2CM6tJvn2kqPXwnXO/d3rAQYiyoIm180VsO8PRX6Rpk=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -279,8 +279,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
+golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
 golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
 golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

internal/api/rest.go

@@ -1423,8 +1423,6 @@ func (api *RestApi) updateConfiguration(rw http.ResponseWriter, r *http.Request)
 	rw.Header().Set("Content-Type", "text/plain")
 	key, value := r.FormValue("key"), r.FormValue("value")
 
-	// fmt.Printf("REST > KEY: %#v\nVALUE: %#v\n", key, value)
-
 	if err := repository.GetUserCfgRepo().UpdateConfig(key, value, repository.GetUserFromContext(r.Context())); err != nil {
 		http.Error(rw, err.Error(), http.StatusUnprocessableEntity)
 		return

internal/config/config.go

@@ -7,9 +7,9 @@ package config
 import (
 	"bytes"
 	"encoding/json"
-	"log"
 	"os"
 
+	"github.com/ClusterCockpit/cc-backend/pkg/log"
 	"github.com/ClusterCockpit/cc-backend/pkg/schema"
 )
 
@@ -53,20 +53,20 @@ func Init(flagConfigFile string) {
 	raw, err := os.ReadFile(flagConfigFile)
 	if err != nil {
 		if !os.IsNotExist(err) {
-			log.Fatalf("CONFIG ERROR: %v", err)
+			log.Abortf("Config Init: Could not read config file '%s'.\nError: %s\n", flagConfigFile, err.Error())
 		}
 	} else {
 		if err := schema.Validate(schema.Config, bytes.NewReader(raw)); err != nil {
-			log.Fatalf("Validate config: %v\n", err)
+			log.Abortf("Config Init: Could not validate config file '%s'.\nError: %s\n", flagConfigFile, err.Error())
 		}
 		dec := json.NewDecoder(bytes.NewReader(raw))
 		dec.DisallowUnknownFields()
 		if err := dec.Decode(&Keys); err != nil {
-			log.Fatalf("could not decode: %v", err)
+			log.Abortf("Config Init: Could not decode config file '%s'.\nError: %s\n", flagConfigFile, err.Error())
 		}
 
 		if Keys.Clusters == nil || len(Keys.Clusters) < 1 {
-			log.Fatal("At least one cluster required in config!")
+			log.Abort("Config Init: At least one cluster required in config. Exited with error.")
 		}
 	}
 }