Virtual host style S3 requests (minio#5095)

Author: Krishna Srinivas

cmd/admin-handlers.go

@@ -985,7 +985,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	var config serverConfigV19
+	var config serverConfigV20
 	err = json.Unmarshal(configBytes, &config)
 
 	if err != nil {

cmd/api-errors.go

@@ -162,6 +162,7 @@ const (
 	ErrServerNotInitialized
 	ErrOperationTimedOut
 	ErrPartsSizeUnequal
+	ErrInvalidRequest
 	// Add new extended error codes here.
 	// Please open a https://github.com/minio/minio/issues before adding
 	// new error codes here.
@@ -731,6 +732,15 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	// Generic Invalid-Request error. Should be used for response errors only for unlikely
+	// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
+	// should be used to log the error at the source of the error for debugging purposes.
+	ErrInvalidRequest: {
+		Code:           "InvalidRequest",
+		Description:    "Invalid Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
 	// Add your error structure here.
 }
 

cmd/api-router.go

@@ -16,9 +16,8 @@
 
 package cmd
 
-import (
-	router "github.com/gorilla/mux"
-)
+import router "github.com/gorilla/mux"
+import "net/http"
 
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
@@ -34,70 +33,75 @@ func registerAPIRouter(mux *router.Router) {
 
 	// API Router
 	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+	var routers []*router.Router
+	if globalDomainName != "" {
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
+	}
+	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	// Bucket router
-	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
-
-	/// Object operations
-
-	// HeadObject
-	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
-	// CopyObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// PutObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// ListObjectPxarts
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
-	// CompleteMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-	// NewMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
-	// AbortMultipartUpload
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-	// GetObject
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
-	// CopyObject
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
-	// PutObject
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
-	// DeleteObject
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
-
-	/// Bucket operations
+	for _, bucket := range routers {
+		// Object operations
+		// HeadObject
+		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		// CopyObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// PutObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// ListObjectPxarts
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		// CompleteMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// NewMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		// AbortMultipartUpload
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObject
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		// CopyObject
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		// PutObject
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		// DeleteObject
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
-	// GetBucketLocation
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
-	// GetBucketPolicy
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
-	// GetBucketNotification
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
-	// ListenBucketNotification
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
-	// ListMultipartUploads
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
-	// ListObjectsV2
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
-	// ListObjectsV1 (Legacy)
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
-	// PutBucketPolicy
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
-	// PutBucketNotification
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
-	// PutBucket
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
-	// HeadBucket
-	bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
-	// PostPolicy
-	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
-	// DeleteMultipleObjects
-	bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler))
-	// DeleteBucketPolicy
-	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
-	// DeleteBucket
-	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		/// Bucket operations
+		// GetBucketLocation
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		// GetBucketPolicy
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		// GetBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		// ListenBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		// ListMultipartUploads
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		// ListObjectsV2
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		// ListObjectsV1 (Legacy)
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		// PutBucketPolicy
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		// PutBucketNotification
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		// PutBucket
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		// HeadBucket
+		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		// PostPolicy
+		bucket.Methods("POST").Path("/").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		// DeleteMultipleObjects
+		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		// DeleteBucketPolicy
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+
+	// If none of the routes match.
+	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/auth-handler.go

@@ -127,7 +127,11 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 	if reqAuthType == authTypeAnonymous && policyAction != "" {
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 		sourceIP := getSourceIPAddress(r)
-		return enforceBucketPolicy(bucket, policyAction, r.URL.Path,
+		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+		if err != nil {
+			return ErrInternalError
+		}
+		return enforceBucketPolicy(bucket, policyAction, resource,
 			r.Referer(), sourceIP, r.URL.Query())
 	}
 

cmd/bucket-handlers.go

@@ -24,7 +24,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -441,10 +440,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Make sure that the URL  does not contain object name.
 	bucket := mux.Vars(r)["bucket"]
-	if bucket != filepath.Clean(r.URL.Path[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
-		return
-	}
 
 	// Require Content-Length to be set in the request
 	size := r.ContentLength

cmd/common-main.go

@@ -57,7 +57,7 @@ func initConfig() {
 	// Config file does not exist, we create it fresh and return upon success.
 	if isFile(getConfigFile()) {
 		fatalIf(migrateConfig(), "Config migration failed.")
-		fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
+		fatalIf(loadConfig(), "Unable to load config version: '%s'.", v20)
 	} else {
 		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
 		log.Println("Created minio configuration file successfully at " + getConfigDir())
@@ -117,4 +117,9 @@ func handleCommonEnvVars() {
 	}
 
 	globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
+
+	globalDomainName = os.Getenv("MINIO_DOMAIN")
+	if globalDomainName != "" {
+		globalIsEnvDomainName = true
+	}
 }

cmd/config-migrate.go

@@ -148,7 +148,12 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
-	case v19:
+	case "19":
+		if err = migrateV19ToV20(); err != nil {
+			return err
+		}
+		fallthrough
+	case "20":
 		// No migration needed. this always points to current version.
 		err = nil
 	}
@@ -1479,3 +1484,111 @@ func migrateV18ToV19() error {
 	log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV19ToV20() error {
+	configFile := getConfigFile()
+
+	cv19 := &serverConfigV19{}
+	_, err := quick.Load(configFile, cv19)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
+	}
+	if cv19.Version != "19" {
+		return nil
+	}
+
+	// Copy over fields from V19 into V20 config struct
+	srvConfig := &serverConfigV20{
+		Logger: &loggers{},
+		Notify: &notifier{},
+	}
+	srvConfig.Version = "20"
+	srvConfig.Credential = cv19.Credential
+	srvConfig.Region = cv19.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	srvConfig.Logger.Console = cv19.Logger.Console
+	srvConfig.Logger.File = cv19.Logger.File
+
+	// check and set notifiers config
+	if len(cv19.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		// New deliveryMode parameter is added for AMQP,
+		// default value is already 0, so nothing to
+		// explicitly migrate here.
+		srvConfig.Notify.AMQP = cv19.Notify.AMQP
+	}
+	if len(cv19.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv19.Notify.ElasticSearch
+	}
+	if len(cv19.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv19.Notify.Redis
+	}
+	if len(cv19.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv19.Notify.PostgreSQL
+	}
+	if len(cv19.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
+		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
+	} else {
+		srvConfig.Notify.Kafka = cv19.Notify.Kafka
+	}
+	if len(cv19.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv19.Notify.NATS
+	}
+	if len(cv19.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
+		srvConfig.Notify.Webhook["1"] = webhookNotify{}
+	} else {
+		srvConfig.Notify.Webhook = cv19.Notify.Webhook
+	}
+	if len(cv19.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
+		srvConfig.Notify.MySQL["1"] = mySQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv19.Notify.MySQL
+	}
+	if len(cv19.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
+		srvConfig.Notify.MQTT["1"] = mqttNotify{}
+	} else {
+		srvConfig.Notify.MQTT = cv19.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv19.Browser
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
+	}
+
+	log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
+	return nil
+}

cmd/config-migrate_test.go

@@ -169,7 +169,7 @@ func TestServerConfigMigrateV2toV19(t *testing.T) {
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := v19
+	expectedVersion := v20
 	if serverConfig.Version != expectedVersion {
 		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
 	}

cmd/config-old.go

@@ -472,3 +472,21 @@ type serverConfigV18 struct {
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
+
+// serverConfigV19 server configuration version '19' which is like
+// version '18' except it adds support for MQTT notifications.
+type serverConfigV19 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}