diff --git a/.github/workflows/it-tests.yml b/.github/workflows/it-tests.yml index 189bf757..de6a7c25 100644 --- a/.github/workflows/it-tests.yml +++ b/.github/workflows/it-tests.yml @@ -16,7 +16,7 @@ jobs: java-version: "17" cache: sbt - name: "Starting dependent containers for testing" - run: docker-compose up -d keycloak ceph postgres-server mariadb ranger-admin rokku-sts + run: docker-compose up -d keycloak ceph postgres-server redis ranger-admin rokku-sts - name: "Install aws cli" run: pip install --user awscli==1.18.222 - name: "Compile the project while containers are starting up" diff --git a/dev-setup/ranger/resources/policy/bucket-create-s3.json b/dev-setup/ranger/resources/policy/bucket-create-s3.json new file mode 100644 index 00000000..79753710 --- /dev/null +++ b/dev-setup/ranger/resources/policy/bucket-create-s3.json @@ -0,0 +1,41 @@ +{ + "service": "testservice", + "name": "bucket management", + "description": "Create/Delete buckets", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "rokkuadmin" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "isEnabled": true, + "version": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/policy/deny-subdir-s3.json b/dev-setup/ranger/resources/policy/deny-subdir-s3.json new file mode 100644 index 00000000..cc643139 --- /dev/null +++ b/dev-setup/ranger/resources/policy/deny-subdir-s3.json @@ -0,0 +1,45 @@ +{ + "service": "testservice", + "name": "testuser_deny_subdir", + "description": "FOR TESTING PURPOSES, Deny access for testuser to subfolder of demobucket", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/demobucket/subdir" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [], + "denyPolicyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "testuser" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "isEnabled": true, + "version": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/policy/home-read-s3.json b/dev-setup/ranger/resources/policy/home-read-s3.json new file mode 100644 index 00000000..5c7d4982 --- /dev/null +++ b/dev-setup/ranger/resources/policy/home-read-s3.json @@ -0,0 +1,41 @@ +{ + "service": "testservice", + "name": "home_read", + "description": "All user can read the home dir no recursive", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/home" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + } + ], + "users": [ + "{USER}" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "isEnabled": true, + "version": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/policy/homedir-s3.json b/dev-setup/ranger/resources/policy/homedir-s3.json new file mode 100644 index 00000000..790514a1 --- /dev/null +++ b/dev-setup/ranger/resources/policy/homedir-s3.json @@ -0,0 +1,45 @@ +{ + "service": "testservice", + "name": "home_dirs", + "description": "FOR TESTING PURPOSES, Allow access for testuser to home subfolder", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/home/{USER}" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "{USER}" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "isEnabled": true, + "version": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/policy/s3.json b/dev-setup/ranger/resources/policy/s3.json new file mode 100644 index 00000000..a78ff0a5 --- /dev/null +++ b/dev-setup/ranger/resources/policy/s3.json @@ -0,0 +1,86 @@ +{ + "service": "testservice", + "name": "testpolicy", + "description": "FOR TESTING PURPOSES, allow all access to demobucket for a test user", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/demobucket" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + } + ], + "users": [ + "testuser" + ], + "groups": [ + "testgroup" + ], + "conditions": [ + { + "type": "cidrAllUserIPs", + "values": [ + "*" + ] + } + ], + "delegateAdmin": false + } + ], + "denyPolicyItems": [ + { + "accesses": [ + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "testuser" + ], + "groups": [ + "testgroup" + ], + "conditions": [], + "delegateAdmin": false + }, + { + "accesses": [ + { + "type": "read", + "isAllowed": true + } + ], + "users": [ + "testuser" + ], + "groups": [], + "conditions": [ + { + "type": "cidrAnyUserIPs", + "values": [ + "1.2.3.4/32" + ] + } + ], + "delegateAdmin": false + } + ], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "policyLabels": [], + "isEnabled": true, + "version": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/policy/shared-s3.json b/dev-setup/ranger/resources/policy/shared-s3.json new file mode 100644 index 00000000..f0d225cb --- /dev/null +++ b/dev-setup/ranger/resources/policy/shared-s3.json @@ -0,0 +1,50 @@ +{ + "service": "testservice", + "name": "shared bucket", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/shared" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "{USER}" + ], + "groups": [ + "role_test" + ], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "id": 5, + "isEnabled": true, + "version": 5 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/service/s3.json b/dev-setup/ranger/resources/service/s3.json new file mode 100644 index 00000000..19893d96 --- /dev/null +++ b/dev-setup/ranger/resources/service/s3.json @@ -0,0 +1,12 @@ +{ + "isEnabled": true, + "version": 1, + "type": "s3", + "name": "testservice", + "description": "FOR TESTING PURPOSES, test service", + "configs": { + "endpoint": "http://ceph:8010", + "password": "secretkey", + "accesskey": "accesskey" + } +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/servicedef/s3.json b/dev-setup/ranger/resources/servicedef/s3.json new file mode 100644 index 00000000..2870418b --- /dev/null +++ b/dev-setup/ranger/resources/servicedef/s3.json @@ -0,0 +1,98 @@ +{ + "name": "s3", + "label": "S3 buckets and objects", + "description": "S3 buckets and objects", + "implClass": "com.ing.ranger.s3.RangerServiceS3", + "version": 1, + "isEnabled": 1, + "resources": [ + { + "itemId": 1, + "name": "path", + "type": "path", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": true, + "ignoreCase": true + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "S3 Bucket", + "description": "S3 Bucket" + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "read", + "label": "read" + }, + { + "itemId": 2, + "name": "write", + "label": "write" + } + ], + "configs": [ + { + "itemId": 1, + "name": "endpoint", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "S3 Endpoint" + }, + { + "itemId": 2, + "name": "accesskey", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Access key" + }, + { + "itemId": 3, + "name": "password", + "type": "password", + "subType": "", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Secret key" + } + ], + "enums": [], + "contextEnrichers": [], + "policyConditions": [ + { + "itemId": 1, + "name": "cidrAllUserIPs", + "label": "All user IP addresses within any cidr range?", + "description": "All user IP addresses within any cidr range?", + "evaluator": "com.ing.wbaa.ranger.plugin.conditionevaluator.AllIpCidrMatcher", + "evaluatorOptions": {} + }, + { + "itemId": 2, + "name": "cidrAnyUserIPs", + "label": "Any user IP address within any cidr range?", + "description": "Any user IP address within any cidr range?", + "evaluator": "com.ing.wbaa.ranger.plugin.conditionevaluator.AnyIpCidrMatcher", + "evaluatorOptions": {} + } + ] +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/users/rokkuadmin.json b/dev-setup/ranger/resources/users/rokkuadmin.json new file mode 100644 index 00000000..70736f25 --- /dev/null +++ b/dev-setup/ranger/resources/users/rokkuadmin.json @@ -0,0 +1,13 @@ +{ + "loginId": "rokkuadmin", + "name": "rokkuadmin", + "password": "password123", + "firstName": "firstname", + "lastName": "lastname", + "publicScreenName": "rokkuadmin", + "userRoleList": [ + "ROLE_USER" + ], + "userPermList": [], + "groupPermissions": [] +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/users/testgroup.json b/dev-setup/ranger/resources/users/testgroup.json new file mode 100644 index 00000000..b61e2931 --- /dev/null +++ b/dev-setup/ranger/resources/users/testgroup.json @@ -0,0 +1,5 @@ +{ + "name": "testgroup", + "description": "", + "isVisible": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/users/testrole.json b/dev-setup/ranger/resources/users/testrole.json new file mode 100644 index 00000000..05a686e7 --- /dev/null +++ b/dev-setup/ranger/resources/users/testrole.json @@ -0,0 +1,5 @@ +{ + "name": "role_test", + "description": "", + "isVisible": 1 +} \ No newline at end of file diff --git a/dev-setup/ranger/resources/users/testuser.json b/dev-setup/ranger/resources/users/testuser.json new file mode 100644 index 00000000..1286f657 --- /dev/null +++ b/dev-setup/ranger/resources/users/testuser.json @@ -0,0 +1,13 @@ +{ + "loginId": "testuser", + "name": "testuser", + "password": "password123", + "firstName": "firstname", + "lastName": "lastname", + "publicScreenName": "testuser", + "userRoleList": [ + "ROLE_USER" + ], + "userPermList": [], + "groupPermissions": [] +} \ No newline at end of file diff --git a/dev-setup/ranger/scripts/setup.sh b/dev-setup/ranger/scripts/setup.sh new file mode 100755 index 00000000..ad9cb094 --- /dev/null +++ b/dev-setup/ranger/scripts/setup.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +sleep 10 + +cd $RANGER_HOME +./setup.sh + +RESOURCES_PATH="/setup/resources" +RANGER_URL="http://localhost:6080" +USERNAME="admin" +PASSWORD="admin" + +/opt/ranger-admin/ews/ranger-admin-services.sh start + +set -e +while [ $(curl --user $USERNAME:$PASSWORD -o -I -L -s -w "%{http_code}" "$RANGER_URL/service/public/v2/api/servicedef/1") -ne 200 ] +do + echo "Waiting for ranger endpoint to be reachable..." + sleep 2 +done + + +echo "Ranger service is reachable!" + +# Create users: +for p in $(find "$RESOURCES_PATH/users/" -name *.json ); do + echo -e "\n\n- Creating user resource for: $p" + curl -i -X POST --user $USERNAME:$PASSWORD "$RANGER_URL/service/xusers/secure/users" -H "Content-Type: application/json" --data "@$p" +done + +#Remove default Hive/HDFS service def +curl -i -X DELETE --user $USERNAME:$PASSWORD "$RANGER_URL/service/public/v2/api/servicedef/1" +curl -i -X DELETE --user $USERNAME:$PASSWORD "$RANGER_URL/service/public/v2/api/servicedef/3" + +#Create service defs +for p in $(find "$RESOURCES_PATH/servicedef/" -name *.json ); do + echo -e "\n\n- Creating servicedef resource for: $p" + curl -i -X POST --user $USERNAME:$PASSWORD "$RANGER_URL/service/public/v2/api/servicedef" -H "Content-Type: application/json" --data "@$p" +done + +#Create services +for p in $(find "$RESOURCES_PATH/service/" -name *.json ); do + echo -e "\n\n- Creating service resource for: $p" + curl -i -X POST --user $USERNAME:$PASSWORD "$RANGER_URL/service/public/v2/api/service" -H "Content-Type: application/json" --data "@$p" +done + +#Create policy +for p in $(find "$RESOURCES_PATH/policy/" -name *.json ); do + echo -e "\n\n- Creating policy resource for: $p" + curl -i -X POST --user $USERNAME:$PASSWORD "$RANGER_URL/service/public/v2/api/policy" -H "Content-Type: application/json" --data "@$p" +done + +echo -e "\n\nRanger setup completed" + +tail -f /dev/null diff --git a/docker-compose.yml b/docker-compose.yml index 492a130f..7cfe3bec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,17 +22,20 @@ services: image: wbaa/rokku-dev-apache-ranger-postgres:0.0.21 ranger-admin: - image: wbaa/rokku-dev-apache-ranger:0.0.21 + image: wbaa/rokku-dev-apache-ranger:2.2.1-genesis-dev stdin_open: true tty: true depends_on: - "postgres-server" - "ceph" + volumes: + - ./dev-setup/ranger:/setup + entrypoint: /setup/scripts/setup.sh ports: - "6080:6080" rokku-sts: - image: wbaa/rokku-sts:0.3.4 + image: wbaa/rokku-sts:v1.0.5 environment: - STS_HOST=0.0.0.0 - STS_PORT=12345 @@ -40,12 +43,13 @@ services: - KEYCLOAK_URL=http://keycloak:8080 - KEYCLOAK_CHECK_REALM_URL=false - KEYCLOAK_CHECK_ISSUER_FOR_LIST=sts-rokku - - MARIADB_URL=jdbc:mysql:loadbalance://mariadb:3306,mariadb:3306/rokku + - REDIS_HOST=redis + - REDIS_PORT=6379 ports: - "12345:12345" depends_on: - "keycloak" - - "mariadb" + - "redis" keycloak: image: wbaa/rokku-dev-keycloak:0.0.8 @@ -56,12 +60,13 @@ services: ports: - "8080:8080" - mariadb: - image: wbaa/rokku-dev-mariadb:0.0.8 + redis: + image: redislabs/redisearch environment: - - MYSQL_ROOT_PASSWORD=admin + - TZ=Europe/Amsterdam + command: "redis-server --requirepass password --loadmodule '/usr/lib/redis/modules/redisearch.so'" ports: - - 3307:3306 + - 6379:6379 zookeeper: image: confluentinc/cp-zookeeper:5.4.0 @@ -88,4 +93,3 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - diff --git a/waitForContainerSetup.sh b/waitForContainerSetup.sh old mode 100644 new mode 100755 index 5300ef60..820cfd72 --- a/waitForContainerSetup.sh +++ b/waitForContainerSetup.sh @@ -11,7 +11,7 @@ function cephIsReady() { docker-compose logs ceph | grep "* Running on http://\[::\]:5000/" } function rangerAdminIsReady() { - docker-compose logs ranger-admin | grep "Policy created" + docker-compose logs ranger-admin | grep "Ranger setup completed" } function rokkuStsIsReady() { docker-compose logs rokku-sts | grep "Sts service started listening:" @@ -19,8 +19,8 @@ function rokkuStsIsReady() { function keycloakIsReady() { docker-compose logs keycloak | grep "Admin console listening" } -function mariadbIsReady() { - docker-compose logs mariadb | grep "Version: '10.3.9-MariaDB-1:10.3.9+maria~bionic' socket: '/var/run/mysqld/mysqld.sock' port: 3306 mariadb.org binary distribution" +function redisIsReady() { + docker-compose logs redis | grep "Ready to accept connections" } function waitUntilServiceIsReady() { @@ -44,5 +44,5 @@ waitUntilServiceIsReady rokkuStsIsReady "Rokku STS" waitUntilServiceIsReady cephIsReady "Ceph" waitUntilServiceIsReady rangerAdminIsReady "Ranger Admin" waitUntilServiceIsReady keycloakIsReady "Keycloack" -waitUntilServiceIsReady mariadbIsReady "MariaDB" +waitUntilServiceIsReady redisIsReady "Redis"