[BACK-2416] Add read-only mode for services that doesn't create indexes.

auth/client/client_test.go

@@ -65,6 +65,7 @@ var _ = Describe("Client", func() {
 			config.Config.UserAgent = testHttp.NewUserAgent()
 			config.Config.ServiceSecret = authTest.NewServiceSecret()
 			config.ExternalConfig.Address = testHttp.NewAddress()
+			config.ExternalConfig.PathPrefix = "auth"
 			config.ExternalConfig.UserAgent = testHttp.NewUserAgent()
 			config.ExternalConfig.ServerSessionTokenSecret = serverTokenSecret
 			config.ExternalConfig.ServerSessionTokenTimeout = time.Duration(serverTokenTimeout) * time.Second
@@ -126,6 +127,7 @@ var _ = Describe("Client", func() {
 			config.Config.UserAgent = testHttp.NewUserAgent()
 			config.Config.ServiceSecret = authTest.NewServiceSecret()
 			config.ExternalConfig.Address = server.URL()
+			config.ExternalConfig.PathPrefix = "auth"
 			config.ExternalConfig.UserAgent = testHttp.NewUserAgent()
 			config.ExternalConfig.ServerSessionTokenSecret = serverTokenSecret
 			authorizeAs = platform.AuthorizeAsService

auth/client/external.go

@@ -68,6 +68,8 @@ type ExternalConfig struct {
 	*platform.Config
 	ServerSessionTokenSecret  string        `envconfig:"TIDEPOOL_AUTH_CLIENT_EXTERNAL_SERVER_SESSION_TOKEN_SECRET"`
 	ServerSessionTokenTimeout time.Duration `envconfig:"TIDEPOOL_AUTH_CLIENT_EXTERNAL_SERVER_SESSION_TOKEN_TIMEOUT" default:"1h"`
+	// PathPrefix to prepend to the path of any service calls (if any).
+	PathPrefix string `envconfig:"TIDEPOOL_AUTH_CLIENT_EXTERNAL_PATH_PREFIX" default:"auth"`
 }
 
 func NewExternalConfig() *ExternalConfig {
@@ -107,6 +109,7 @@ type External struct {
 	serverSessionTokenMutex   sync.Mutex
 	serverSessionTokenSafe    string
 	closingChannel            chan chan bool
+	PathPrefix                string // PathPrefix is the prefix to prepend to all external URL path calls to the auth service (if any)
 }
 
 func NewExternal(cfg *ExternalConfig, authorizeAs platform.AuthorizeAs, name string, lgr log.Logger) (*External, error) {
@@ -135,6 +138,7 @@ func NewExternal(cfg *ExternalConfig, authorizeAs platform.AuthorizeAs, name str
 		name:                      name,
 		serverSessionTokenSecret:  cfg.ServerSessionTokenSecret,
 		serverSessionTokenTimeout: cfg.ServerSessionTokenTimeout,
+		PathPrefix:                cfg.PathPrefix,
 	}, nil
 }
 
@@ -200,7 +204,7 @@ func (e *External) ValidateSessionToken(ctx context.Context, token string) (requ
 		IsServer bool
 		UserID   string
 	}
-	if err := e.client.RequestData(ctx, "GET", e.client.ConstructURL("auth", "token", token), nil, nil, &result); err != nil {
+	if err := e.client.RequestData(ctx, "GET", e.client.ConstructURL(e.PathPrefix, "token", token), nil, nil, &result); err != nil {
 		return nil, err
 	}
 
@@ -329,7 +333,7 @@ func (e *External) refreshServerSessionToken() error {
 	e.logger.Debug("Refreshing server session token")
 
 	requestMethod := "POST"
-	requestURL := e.client.ConstructURL("auth", "serverlogin")
+	requestURL := e.client.ConstructURL(e.PathPrefix, "serverlogin")
 	request, err := http.NewRequest(requestMethod, requestURL, nil)
 	if err != nil {
 		return errors.Wrapf(err, "unable to create new request for %s %s", requestMethod, requestURL)

auth/service/service/service.go

@@ -129,7 +129,7 @@ func (s *Service) Initialize(provider application.Provider) error {
 	if err := s.initializeTwiistServiceAccountAuthorizer(); err != nil {
 		return err
 	}
-	return s.initializeUserEventsHandler()
+	return s.initializeUserEventsHandler(provider)
 }
 
 func (s *Service) Terminate() {
@@ -468,13 +468,20 @@ func (s *Service) terminateAuthClient() {
 	}
 }
 
-func (s *Service) initializeUserEventsHandler() error {
+func (s *Service) initializeUserEventsHandler(provider application.Provider) error {
 	s.Logger().Debug("Initializing user events handler")
 
-	ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger())
-	handler := authEvents.NewUserDataDeletionHandler(ctx, s.authClient)
-	handlers := []eventsCommon.EventHandler{handler}
-	runner := events.NewRunner(handlers)
+	var runner events.Runner
+
+	configReporter := provider.ConfigReporter().WithScopes("user", "events", "handler")
+	if configReporter.GetWithDefault("disable", "") != "true" {
+		ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger())
+		handler := authEvents.NewUserDataDeletionHandler(ctx, s.authClient)
+		handlers := []eventsCommon.EventHandler{handler}
+		runner = events.NewRunner(handlers)
+	} else {
+		runner = events.NewNoopRunner()
+	}
 
 	if err := runner.Initialize(); err != nil {
 		return errors.Wrap(err, "unable to initialize events runner")

auth/service/service/service_test.go

@@ -49,7 +49,7 @@ var _ = Describe("Service", func() {
 			server = NewServer()
 			server.AppendHandlers(
 				CombineHandlers(
-					VerifyRequest("POST", "/auth/serverlogin"),
+					VerifyRequest("POST", "/serverlogin"), // by default the path prefix is empty to the auth service unless set in the env var TIDEPOOL_AUTH_CLIENT_EXTERNAL_PATH_PREFIX
 					VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput),
 					VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret),
 					VerifyBody(nil),

blob/service/service.go

@@ -69,7 +69,7 @@ func (s *Service) Initialize(provider application.Provider) error {
 	if err := s.initializeBlobClient(); err != nil {
 		return err
 	}
-	if err := s.initializeUserEventsHandler(); err != nil {
+	if err := s.initializeUserEventsHandler(provider); err != nil {
 		return err
 	}
 	return s.initializeRouter()
@@ -211,13 +211,20 @@ func (s *Service) terminateDeviceLogsUnstructuredStore() {
 	}
 }
 
-func (s *Service) initializeUserEventsHandler() error {
+func (s *Service) initializeUserEventsHandler(provider application.Provider) error {
 	s.Logger().Debug("Initializing user events handler")
 
-	ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger())
-	handler := blobEvents.NewUserDataDeletionHandler(ctx, s.blobClient)
-	handlers := []eventsCommon.EventHandler{handler}
-	runner := events.NewRunner(handlers)
+	var runner events.Runner
+
+	configReporter := provider.ConfigReporter().WithScopes("user", "events", "handler")
+	if configReporter.GetWithDefault("disable", "") != "true" {
+		ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger())
+		handler := blobEvents.NewUserDataDeletionHandler(ctx, s.blobClient)
+		handlers := []eventsCommon.EventHandler{handler}
+		runner = events.NewRunner(handlers)
+	} else {
+		runner = events.NewNoopRunner()
+	}
 
 	if err := runner.Initialize(); err != nil {
 		return errors.Wrap(err, "unable to initialize events runner")

blob/service/service_test.go

@@ -47,7 +47,7 @@ var _ = Describe("Service", func() {
 			server = NewServer()
 			server.AppendHandlers(
 				CombineHandlers(
-					VerifyRequest(http.MethodPost, "/auth/serverlogin"),
+					VerifyRequest(http.MethodPost, "/serverlogin"),
 					VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput),
 					VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret),
 					VerifyBody(nil),

client/client.go

@@ -58,7 +58,11 @@ func NewWithErrorParser(cfg *Config, errorResponseParser ErrorResponseParser) (*
 func (c *Client) ConstructURL(paths ...string) string {
 	segments := []string{}
 	for _, path := range paths {
-		segments = append(segments, url.PathEscape(strings.Trim(path, "/")))
+		escapedPath := url.PathEscape(strings.Trim(path, "/"))
+		if escapedPath == "" {
+			continue
+		}
+		segments = append(segments, escapedPath)
 	}
 	return fmt.Sprintf("%s/%s", strings.TrimRight(c.address, "/"), strings.Join(segments, "/"))
 }

client/config.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"fmt"
 	"net/url"
 
 	"github.com/kelseyhightower/envconfig"
@@ -37,7 +38,7 @@ func (c *Config) Validate() error {
 	if c.Address == "" {
 		return errors.New("address is missing")
 	} else if _, err := url.Parse(c.Address); err != nil {
-		return errors.New("address is invalid")
+		return fmt.Errorf("address is invalid: %w", err)
 	}
 
 	return nil

client/config_test.go

@@ -86,7 +86,7 @@ var _ = Describe("Config", func() {
 
 				It("returns an error if the address is not a parseable URL", func() {
 					cfg.Address = "Not%Parseable"
-					Expect(cfg.Validate()).To(MatchError("address is invalid"))
+					Expect(cfg.Validate()).To(MatchError("address is invalid: parse \"Not%Parseable\": invalid URL escape \"%Pa\""))
 				})
 
 				It("returns success", func() {

data/service/api/v1/datasets_data_create.go

@@ -149,7 +149,8 @@ func CollectProvenanceInfo(ctx context.Context, req *rest.Request, authDetails r
 	if token != "" && shouldHaveJWT(authDetails) {
 		claims := &TokenClaims{}
 		if _, _, err := jwt.NewParser().ParseUnverified(token, claims); err != nil {
-			lgr.WithError(err).Warn("Unable to parse access token for provenance")
+			// temp comment out because makes realtime viewing rough
+			// lgr.WithError(err).Warn("Unable to parse access token for provenance")
 		} else {
 			provenance.ClientID = claims.ClientID
 		}

data/service/service/standard.go

@@ -90,7 +90,7 @@ func (s *Standard) Initialize(provider application.Provider) error {
 	if err := s.initializeDataSourceClient(); err != nil {
 		return err
 	}
-	if err := s.initializeUserEventsHandler(); err != nil {
+	if err := s.initializeUserEventsHandler(provider); err != nil {
 		return err
 	}
 	if err := s.initializeTwiistServiceAccountAuthorizer(); err != nil {
@@ -430,14 +430,22 @@ func (s *Standard) initializeServer() error {
 	return nil
 }
 
-func (s *Standard) initializeUserEventsHandler() error {
+func (s *Standard) initializeUserEventsHandler(provider application.Provider) error {
 	s.Logger().Debug("Initializing user events handler")
 	sarama.Logger = log.New(os.Stdout, "SARAMA ", log.LstdFlags|log.Lshortfile)
 
-	ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger())
-	handler := dataEvents.NewUserDataDeletionHandler(ctx, s.dataStore, s.dataSourceStructuredStore)
-	handlers := []eventsCommon.EventHandler{handler}
-	runner := events.NewRunner(handlers)
+	var runner events.Runner
+
+	configReporter := provider.ConfigReporter().WithScopes("user", "events", "handler")
+	if configReporter.GetWithDefault("disable", "") != "true" {
+		ctx := logInternal.NewContextWithLogger(context.Background(), s.Logger())
+		handler := dataEvents.NewUserDataDeletionHandler(ctx, s.dataStore, s.dataSourceStructuredStore)
+		handlers := []eventsCommon.EventHandler{handler}
+		runner = events.NewRunner(handlers)
+	} else {
+		runner = events.NewNoopRunner()
+	}
+
 	if err := runner.Initialize(); err != nil {
 		return errors.Wrap(err, "unable to initialize user events handler runner")
 	}

data/store/mongo/mongo_data_set.go

@@ -40,16 +40,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
 			Options: options.Index().
 				SetName("UserIdTypeWeighted_v2"),
 		},
-		{
-			Keys: bson.D{
-				{Key: "origin.id", Value: 1},
-				{Key: "type", Value: 1},
-				{Key: "deletedTime", Value: -1},
-				{Key: "_active", Value: 1},
-			},
-			Options: options.Index().
-				SetName("OriginId"),
-		},
 		{
 			Keys: bson.D{
 				{Key: "uploadId", Value: 1},
@@ -330,14 +320,14 @@ func (d *DataSetRepository) GetDataSetsForUserByID(ctx context.Context, userID s
 		SetSort(bson.M{"createdTime": -1})
 	cursor, err := d.Find(ctx, selector, opts)
 
-	loggerFields := log.Fields{"userId": userID, "dataSetsCount": len(dataSets), "duration": time.Since(now) / time.Microsecond}
-	log.LoggerFromContext(ctx).WithFields(loggerFields).WithError(err).Debug("GetDataSetsForUserByID")
-
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to get data sets for user by id")
 	}
 
-	if err = cursor.All(ctx, &dataSets); err != nil {
+	err = cursor.All(ctx, &dataSets)
+	loggerFields := log.Fields{"userId": userID, "dataSetsCount": len(dataSets), "duration": time.Since(now) / time.Microsecond}
+	log.LoggerFromContext(ctx).WithFields(loggerFields).WithError(err).Debug("GetDataSetsForUserByID")
+	if err != nil {
 		return nil, errors.Wrap(err, "unable to decode data sets for user by id")
 	}
 

data/store/mongo/mongo_datum.go

@@ -471,7 +471,8 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co
 			"modifiedTime":      timestamp,
 		}
 		unset := bson.M{}
-		updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset))
+		opts := options.Update()
+		updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
 	}
 
 	loggerFields := log.Fields{"userId": dataSet.UserID, "deviceId": *dataSet.DeviceID, "updateInfo": updateInfo, "duration": time.Since(now) / time.Microsecond}

data/store/mongo/mongo_test.go

@@ -371,10 +371,6 @@ var _ = Describe("Mongo", Label("mongodb", "slow", "integration"), func() {
 						"Key":  Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")),
 						"Name": Equal("UserIdTypeWeighted_v2"),
 					}),
-					MatchFields(IgnoreExtras, Fields{
-						"Key":  Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
-						"Name": Equal("OriginId"),
-					}),
 					MatchFields(IgnoreExtras, Fields{
 						"Key":  Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
 						"Name": Equal("UploadId"),

env.sh

@@ -32,6 +32,7 @@ export TIDEPOOL_USER_CLIENT_ADDRESS="http://localhost:8009"
 
 export TIDEPOOL_AUTH_CLIENT_EXTERNAL_ADDRESS="http://localhost:8009"
 export TIDEPOOL_AUTH_CLIENT_EXTERNAL_SERVER_SESSION_TOKEN_SECRET="This needs to be the same secret everywhere. YaHut75NsK1f9UKUXuWqxNN0RUwHFBCy"
+export TIDEPOOL_AUTH_CLIENT_EXTERNAL_PATH_PREFIX="auth"
 
 export TIDEPOOL_AUTH_SERVICE_SERVER_ADDRESS=":9222"
 export TIDEPOOL_BLOB_SERVICE_SERVER_ADDRESS=":9225"

events/events.go

@@ -47,3 +47,28 @@ func (r *runner) Terminate() error {
 	}
 	return nil
 }
+
+type noopRunner struct {
+	terminate chan struct{}
+}
+
+func (n *noopRunner) Initialize() error {
+	n.terminate = make(chan struct{}, 0)
+	return nil
+}
+
+func (n *noopRunner) Run() error {
+	<-n.terminate
+	return nil
+}
+
+func (n *noopRunner) Terminate() error {
+	n.terminate <- struct{}{}
+	return nil
+}
+
+var _ Runner = &noopRunner{}
+
+func NewNoopRunner() Runner {
+	return &noopRunner{}
+}

platform/config_test.go

@@ -110,7 +110,7 @@ var _ = Describe("Config", func() {
 
 				It("returns an error if the address is not a parseable URL", func() {
 					cfg.Address = "Not%Parseable"
-					Expect(cfg.Validate()).To(MatchError("address is invalid"))
+					Expect(cfg.Validate()).To(MatchError("address is invalid: parse \"Not%Parseable\": invalid URL escape \"%Pa\""))
 				})
 
 				It("returns success", func() {

prescription/application/application_test.go

@@ -53,7 +53,7 @@ var _ = Describe("Application", func() {
 			server = NewServer()
 			server.AppendHandlers(
 				CombineHandlers(
-					VerifyRequest(http.MethodPost, "/auth/serverlogin"),
+					VerifyRequest(http.MethodPost, "/serverlogin"),
 					VerifyHeaderKV("X-Tidepool-Server-Name", *prvdr.NameOutput),
 					VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret),
 					VerifyBody(nil),

service/service/DEPRECATED_service_test.go

@@ -39,7 +39,7 @@ var _ = Describe("DEPRECATEDService", func() {
 			Expect(server).ToNot(BeNil())
 			server.AppendHandlers(
 				CombineHandlers(
-					VerifyRequest("POST", "/auth/serverlogin"),
+					VerifyRequest("POST", "/serverlogin"),
 					VerifyHeaderKV("X-Tidepool-Server-Name", *provider.NameOutput),
 					VerifyHeaderKV("X-Tidepool-Server-Secret", serverSecret),
 					VerifyBody(nil),

store/structured/mongo/config.go

@@ -28,15 +28,16 @@ func LoadConfig() (*Config, error) {
 
 // Config describe parameters need to make a connection to a Mongo database
 type Config struct {
-	Scheme           string        `json:"scheme" envconfig:"TIDEPOOL_STORE_SCHEME"`
-	Addresses        []string      `json:"addresses" envconfig:"TIDEPOOL_STORE_ADDRESSES" required:"true"`
-	TLS              bool          `json:"tls" envconfig:"TIDEPOOL_STORE_TLS" default:"true"`
-	Database         string        `json:"database" envconfig:"TIDEPOOL_STORE_DATABASE" required:"true"`
-	CollectionPrefix string        `json:"collectionPrefix" envconfig:"TIDEPOOL_STORE_COLLECTION_PREFIX"`
-	Username         *string       `json:"-" envconfig:"TIDEPOOL_STORE_USERNAME"`
-	Password         *string       `json:"-" envconfig:"TIDEPOOL_STORE_PASSWORD"`
-	Timeout          time.Duration `json:"timeout" envconfig:"TIDEPOOL_STORE_TIMEOUT" default:"60s"`
-	OptParams        *string       `json:"optParams" envconfig:"TIDEPOOL_STORE_OPT_PARAMS"`
+	Scheme               string        `json:"scheme" envconfig:"TIDEPOOL_STORE_SCHEME"`
+	Addresses            []string      `json:"addresses" envconfig:"TIDEPOOL_STORE_ADDRESSES" required:"true"`
+	TLS                  bool          `json:"tls" envconfig:"TIDEPOOL_STORE_TLS" default:"true"`
+	Database             string        `json:"database" envconfig:"TIDEPOOL_STORE_DATABASE" required:"true"`
+	CollectionPrefix     string        `json:"collectionPrefix" envconfig:"TIDEPOOL_STORE_COLLECTION_PREFIX"`
+	Username             *string       `json:"-" envconfig:"TIDEPOOL_STORE_USERNAME"`
+	Password             *string       `json:"-" envconfig:"TIDEPOOL_STORE_PASSWORD"`
+	Timeout              time.Duration `json:"timeout" envconfig:"TIDEPOOL_STORE_TIMEOUT" default:"60s"`
+	OptParams            *string       `json:"optParams" envconfig:"TIDEPOOL_STORE_OPT_PARAMS"`
+	DisableIndexCreation bool          `json:"disableIndexCreation" envconfig:"TIDEPOOL_DISABLE_INDEX_CREATION"`
 }
 
 // AsConnectionString constructs a MongoDB connection string from a Config