Merge pull request #1137 from mrtamm/develop-task-ownership

Task ownership and access-control

.github/workflows/build.yml

@@ -8,34 +8,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v5
-      with:
-        go-version: 1.22
-
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
-    - name: Cache Funnel binary
-      uses: actions/cache@v3
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
       with:
-        path: ./funnel
-        key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }}
-        restore-keys: |
-          ${{ runner.os }}-funnel-bin-${{ github.ref }}
-          ${{ runner.os }}-funnel-bin-
+        go-version-file: go.mod
 
-    - name: Build Funnel (if cache doesn't exist)
-      run: |
-        if [ ! -f ./funnel ]; then
-          make build
-        fi
-    
-    - name: Cache Funnel binary (after build)
-      uses: actions/cache@v3
-      with:
-        path: ./funnel
-        key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }}
+    - name: Build Funnel
+      run: make build
 
     - name: Upload Funnel binary as artifact
       uses: actions/upload-artifact@v4

.github/workflows/tests.yaml

@@ -12,12 +12,14 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21
+          go-version-file: go.mod
 
-      - uses: actions/checkout@v3
-
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -31,7 +33,7 @@ jobs:
               --skip-dirs "funnel-work-dir" \
               -e '.*bundle.go' -e ".*pb.go" -e ".*pb.gw.go" \
               ./...
-      
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -44,12 +46,13 @@ jobs:
   unitTest:
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
+
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: 1.21
-    - name: Check out code
-      uses: actions/checkout@v2
+        go-version-file: go.mod
 
     - name: Unit Tests
       run: make test-verbose
@@ -61,13 +64,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
+
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: 1.21
-
-    - name: Check out code
-      uses: actions/checkout@v2
+        go-version-file: go.mod
 
     - name: Download funnel bin
       uses: actions/download-artifact@v4
@@ -85,17 +88,19 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
+
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: 1.21
-    - name: Check out code
-      uses: actions/checkout@v2
+        go-version-file: go.mod
 
     - name: Download funnel bin
       uses: actions/download-artifact@v4
       with:
         name: funnel
+
     - name: Badger Test
       run: |
         chmod +x funnel
@@ -105,12 +110,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
+
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: 1.21
-    - name: Check out code
-      uses: actions/checkout@v2
+        go-version-file: go.mod
 
     - name: Download funnel bin
       uses: actions/download-artifact@v4
@@ -126,12 +132,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
+
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: 1.21
-    - name: Check out code
-      uses: actions/checkout@v2
+        go-version-file: go.mod
 
     - name: Download funnel bin
       uses: actions/download-artifact@v4
@@ -144,4 +151,3 @@ jobs:
         make start-generic-s3
         sleep 10
         make test-generic-s3
-          

Makefile

@@ -35,7 +35,7 @@ build:
 	@touch version/version.go
 	@go build -ldflags '$(VERSION_LDFLAGS)' -buildvcs=false .
 
-# Build an unoptimized version of the code for use during debugging 
+# Build an unoptimized version of the code for use during debugging
 # https://go.dev/doc/gdb
 debug:
 	@go install -gcflags=all="-N -l"
@@ -119,7 +119,7 @@ test-verbose:
 
 start-elasticsearch:
 	@docker rm -f funnel-es-test > /dev/null 2>&1 || echo
-	@docker run -d --name funnel-es-test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.elastic.co/elasticsearch/elasticsearch:5.6.3 > /dev/null
+	@docker run -d --name funnel-es-test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" docker.io/elastic/elasticsearch:8.17.1 > /dev/null
 
 test-elasticsearch:
 	@go test ./tests/core/ -funnel-config `pwd`/tests/elastic.config.yml
@@ -140,7 +140,7 @@ test-badger:
 
 start-dynamodb:
 	@docker rm -f funnel-dynamodb-test > /dev/null 2>&1 || echo
-	@docker run -d --name funnel-dynamodb-test -p 18000:8000 docker.io/dwmkerr/dynamodb:38 -sharedDb > /dev/null
+	@docker run -d --name funnel-dynamodb-test -p 18000:8000 docker.io/amazon/dynamodb-local > /dev/null
 
 test-dynamodb:
 	@go test ./tests/core/ -funnel-config `pwd`/tests/dynamo.config.yml
@@ -228,7 +228,7 @@ snapshot: release-dep
 docker:
 	docker build -t quay.io/ohsu-comp-bio/funnel:latest ./
 
-# Create a release on Github using GoReleaser 
+# Create a release on Github using GoReleaser
 release:
 	@go get github.com/buchanae/github-release-notes
 	@goreleaser \

cmd/server/run.go

@@ -269,6 +269,7 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se
 			BasicAuth:        conf.Server.BasicAuth,
 			OidcAuth:         conf.Server.OidcAuth,
 			DisableHTTPCache: conf.Server.DisableHTTPCache,
+			TaskAccess:       conf.Server.TaskAccess,
 			Log:              log,
 			Tasks: &server.TaskService{
 				Name:    conf.Server.ServiceName,

cmd/task/list.go

@@ -18,7 +18,8 @@ func List(server, taskView, pageToken, stateFilter string, tagsFilter []string,
 		return err
 	}
 
-	_, err = getTaskView(taskView)
+	taskViewInt, err := getTaskView(taskView)
+	taskView = tes.View_name[taskViewInt]
 	if err != nil {
 		return err
 	}

compute/hpc_backend.go

@@ -104,11 +104,6 @@ func (b *HPCBackend) Cancel(ctx context.Context, taskID string) error {
 		return err
 	}
 
-	// only cancel tasks in a QUEUED state
-	if task.State != tes.State_QUEUED {
-		return nil
-	}
-
 	backendID := getBackendTaskID(task, b.Name)
 	if backendID == "" {
 		return fmt.Errorf("no %s_id found in metadata for task %s", b.Name, taskID)

config/config.go

@@ -54,6 +54,7 @@ type Config struct {
 type BasicCredential struct {
 	User     string
 	Password string
+	Admin    bool
 }
 
 type OidcAuth struct {
@@ -63,6 +64,7 @@ type OidcAuth struct {
 	RedirectURL      string
 	RequireScope     string
 	RequireAudience  string
+	Admins           []string
 }
 
 // RPCClient describes configuration for gRPC clients
@@ -88,6 +90,12 @@ type Server struct {
 	BasicAuth        []BasicCredential
 	OidcAuth         OidcAuth
 	DisableHTTPCache bool
+
+	// Defines task access and visibility by options:
+	// "All" (default) – all tasks are visible to everyone
+	// "Owner" - tasks are visible to the users who created them
+	// "OwnerOrAdmin" - extends "Owner" by allowing Admin-users see everything
+	TaskAccess string
 }
 
 // HTTPAddress returns the HTTP address based on HostName and HTTPPort
@@ -236,8 +244,13 @@ type MongoDB struct {
 
 // Elastic configures access to an Elasticsearch database.
 type Elastic struct {
-	IndexPrefix string
-	URL         string
+	IndexPrefix  string
+	URL          string
+	Username     string
+	Password     string
+	CloudID      string
+	APIKey       string
+	ServiceToken string
 }
 
 // Kafka configure access to a Kafka topic for task event reading/writing.

config/datastore/README.md

@@ -0,0 +1,26 @@
+# Google Datastore Usage
+
+When Funnel is configured to use the Google Datastore as its database, some
+additional configuration steps need to be taken.
+
+## Datastore Access
+
+Authentication to the Google Datastore needs to be configured through Google
+Cloud CLI as described here:
+https://cloud.google.com/datastore/docs/reference/libraries?hl=en#authentication
+
+## Datastore Indexes
+
+For retrieving a list of tasks, Funnel needs [composite
+indexes](https://cloud.google.com/datastore/docs/concepts/indexes?hl=en) to be
+defined in the Datastore using the Google Cloud CLI and the
+[index.yaml](./index.yaml) file:
+
+```shell
+gcloud datastore indexes create path/to/index.yaml --database='funnel'
+```
+
+Note that it will take a bit of time before the indexes are ready for accepting
+requests. You can see the status of those indexes through the Google Cloud
+console: https://console.cloud.google.com/datastore/databases/ (**Indexes**
+view under the target database).

config/datastore/index.yaml

@@ -0,0 +1,51 @@
+# These index-definitions need to be imported to Datastore (if `database: datastore`)
+# Using Google Cloud CLI: gcloud datastore indexes create path/to/index.yaml
+
+indexes:
+
+- kind: Task
+  properties:
+  - name: Owner
+  - name: State
+  - name: TagStrings
+  - name: CreationTime
+    direction: desc
+
+- kind: Task
+  properties:
+  - name: Owner
+  - name: State
+  - name: CreationTime
+    direction: desc
+
+- kind: Task
+  properties:
+  - name: Owner
+  - name: TagStrings
+  - name: CreationTime
+    direction: desc
+
+- kind: Task
+  properties:
+  - name: Owner
+  - name: CreationTime
+    direction: desc
+
+- kind: Task
+  properties:
+  - name: State
+  - name: TagStrings
+  - name: CreationTime
+    direction: desc
+
+- kind: Task
+  properties:
+  - name: State
+  - name: CreationTime
+    direction: desc
+
+- kind: Task
+  properties:
+  - name: TagStrings
+  - name: CreationTime
+    direction: desc